diff --git a/src/a2a3/runtime/host_build_graph/build_config.py b/src/a2a3/runtime/host_build_graph/build_config.py index 1a9cfd5ca7..35d5036ec5 100644 --- a/src/a2a3/runtime/host_build_graph/build_config.py +++ b/src/a2a3/runtime/host_build_graph/build_config.py @@ -24,7 +24,10 @@ BUILD_CONFIG = { "aicore": {"include_dirs": ["runtime", "common", ".."], "source_dirs": ["aicore", "orchestration"]}, - "aicpu": {"include_dirs": ["runtime", "common", ".."], "source_dirs": ["aicpu", "runtime", "orchestration"]}, + "aicpu": { + "include_dirs": ["runtime", "common", ".."], + "source_dirs": ["aicpu", "runtime", "orchestration", "../../../common/host_build_graph"], + }, "host": { "include_dirs": ["runtime", "common", ".."], "source_dirs": ["host", "runtime/orchestrator_core", "runtime/shared", "orchestration"], diff --git a/src/a2a3/runtime/host_build_graph/docs/GRAPH_EXECUTION.md b/src/a2a3/runtime/host_build_graph/docs/GRAPH_EXECUTION.md index 09b03d97cb..d757e463a4 100644 --- a/src/a2a3/runtime/host_build_graph/docs/GRAPH_EXECUTION.md +++ b/src/a2a3/runtime/host_build_graph/docs/GRAPH_EXECUTION.md @@ -1,387 +1,4 @@ # Graph Execution -Graph Execution is available only in the `host_build_graph` runtime. A Graph is -a composite incore task: it is submitted and completed once like an AIC, AIV, -MIX, or SPMD task, but contains a recorded task DAG. - -Every invocation places exactly one `GRAPH` task in the host task window. The -first invocation records the DAG off the ring — its internal submissions build -host-only node metadata and reserve scratch output buffers instead of consuming -task-window slots — then emits the outer `GRAPH` task from the freshly built -Definition. Later invocations reuse the cached Definition and emit the same one -`GRAPH` task directly. In both cases the device Scheduler expands the saved -topology and dispatches the internal nodes; the Host Orchestrator never submits -those nodes as ring tasks. - -A recording that hits an unsupported construct is discarded and the body re-runs -on the ordinary task-submit path so its work is still submitted; the internal -nodes then occupy the ring only for that one fallback invocation. - -## API - -A Graph uses `CoreTaskArgs`, the existing incore argument type: - -```cpp -void graph_function(const CoreTaskArgs &args, int variant) { - const ChipTensor &input = args.tensor(0).ref(); - const ChipTensor &weight = args.tensor(1).ref(); - const ChipTensor &output = args.tensor(2).ref(); - - const std::array shape{input.shapes[0]}; - TensorCreateInfo intermediate( - shape.data(), static_cast(shape.size()), input.dtype - ); - - CoreTaskArgs matmul_args; - matmul_args.add_input(input, weight); - matmul_args.add_output(intermediate); - matmul_args.copy_scalars_from(args, 0, 1); // current invocation's value - TaskOutputTensors matmul = rt_submit_aic_task( - variant == 0 ? FUNC_MATMUL : FUNC_MATMUL_TRANSPOSED, - matmul_args - ); - - CoreTaskArgs activation_args; - activation_args.add_input(matmul.get_ref(0)); - activation_args.add_output(output); - rt_submit_aiv_task(FUNC_ACTIVATION, activation_args); -} - -void submit_layer(const CoreTaskArgs &args) { - rt_submit_graph(&graph_function, args, /*variant=*/0); -} -``` - -The function pointer is the default Graph identity. Trailing integral, -`float`, `double`, and `bool` construction parameters are forwarded to the -Graph function and hashed by value into the cache key. They are separate from -execution scalars in `CoreTaskArgs`: changing a construction parameter selects a -different Definition rather than patching an existing one. - -An explicit identity is available for call sites that need a stable name: - -```cpp -rt_submit_graph( - GRAPH_KEY("qwen_decoder_layer_v1"), - &graph_function, - args, - /*variant=*/0 -); -``` - -An explicit `GRAPH_KEY` must be unique for every distinct Graph function in an -orchestration callable. The explicit-key overload deliberately excludes the -Graph function pointer from the cache identity so the key remains stable; using -the same key for different functions can select the wrong recorded topology. - -There are no public `GraphArgs`, `GraphBindings`, `Patch`, or `ScalarRef` -types. The boundary is represented by `CoreTaskArgs`. - -Boundary scalars are pass-through bindings. Forward them directly with -`node_args.add_scalar(args.scalar(i))` or `copy_scalars_from(args, i, count)` -so recording can retain their source indices. - -Ordinary C++ value transformations do not retain boundary provenance. Both -`node_args.add_scalar(args.scalar(i) + 1)` and copying `args.scalar(i)` into a -local arithmetic variable before calling `add_scalar` produce an ordinary -static node scalar. That value is stored in the Definition, and later cache -hits reuse the first invocation's value without a warning. The runtime cannot -distinguish such a derived value from an intentional static literal after the -C++ expression has produced a plain arithmetic value. Compute the derived value -before constructing the Graph boundary and pass it as another boundary scalar, -perform the transformation in a kernel, or use a construction parameter when -the value changes the Graph structure. - -Access through a non-const `scalar()` invalidates inherited boundary provenance -conservatively, because returning a mutable reference cannot distinguish a read -from a later write. A Graph containing such an invalidated binding is not -cached, which prevents replay from silently replacing the transformed value -with the unmodified boundary value. - -## Supported dynamic and static data - -- Boundary ChipTensor addresses may change for every invocation. -- Boundary scalar values may change for every invocation. Their count is fixed - by the recorded boundary contract. Unused boundary scalars are allowed and do - not create internal scalar patches. -- A Graph boundary contains at least one ChipTensor. -- Construction parameters are part of Graph identity and may control the - function's task count, kernel selection, or other structural choices. -- Boundary ChipTensor shape, stride, dtype, size, direction, contiguity, and alias - partition must match the first invocation. -- Internal task scalars with no boundary source are fixed Definition data. -- Boundary storage is caller-owned. `INPUT`, `INOUT`, `OUTPUT_EXISTING`, and - `NO_DEP` are supported. A boundary `TensorCreateInfo` tagged `OUTPUT` is not. -- Early-resolve hints apply while recording the first invocation. Replayed - internal nodes use the saved completion topology without the hint. -- A recorded task may depend on a Graph-external producer when that producer - is the creator of a boundary ChipTensor. The outer Graph owns that dependency on - replay; arbitrary cross-boundary explicit dependencies remain unsupported. - -Structural or alias mismatch logs a warning and executes the Graph function -normally for that invocation. It never reuses heap offsets recorded for a -different shape. Debug builds also assert at these unsupported boundaries so -development catches a violated fixed-shape contract immediately; the ordinary -path remains the defensive release-build behavior. - -## Qwen decoder-layer example - -The upper layer packages all ChipTensor I/O in `CoreTaskArgs`; the wrapper has no -separate `hidden`, `weight`, or `output` parameters: - -```cpp -void qwen_decoder_layer(const CoreTaskArgs &args) { - const ChipTensor &hidden = args.tensor(0).ref(); - const ChipTensor &attention_weight = args.tensor(1).ref(); - const ChipTensor &mlp_weight = args.tensor(2).ref(); - const ChipTensor &output = args.tensor(3).ref(); - - const std::array hidden_shape{hidden.shapes[0]}; - TensorCreateInfo attention_out( - hidden_shape.data(), static_cast(hidden_shape.size()), hidden.dtype - ); - - CoreTaskArgs attention_args; - attention_args.add_input(hidden, attention_weight); - attention_args.add_output(attention_out); - attention_args.copy_scalars_from(args, 0, 1); // dynamic token position - TaskOutputTensors attention = - rt_submit_aic_task(FUNC_ATTENTION, attention_args); - - MixedKernels mlp; - mlp.aic_kernel_id = FUNC_MLP_AIC; - mlp.aiv0_kernel_id = FUNC_MLP_AIV; - - CoreTaskArgs mlp_args; - mlp_args.add_input(attention.get_ref(0), mlp_weight); - mlp_args.add_output(output); - rt_submit_task(mlp, mlp_args); -} - -void submit_qwen_decoder_layer(const CoreTaskArgs &args) { - rt_submit_graph(&qwen_decoder_layer, args); -} - -void decode_three_layers( - const std::array &hidden, - const std::array &attention_weight, - const std::array &mlp_weight, - const std::array &output, - const std::array &token_position -) { - for (std::size_t layer = 0; layer < hidden.size(); ++layer) { - CoreTaskArgs args; - args.add_input( - hidden[layer], - attention_weight[layer], - mlp_weight[layer] - ); - args.add_output(output[layer]); - args.add_scalar(token_position[layer]); - submit_qwen_decoder_layer(args); - } -} -``` - -All three layers submit one Graph task each: the first records the sub-DAG off -the ring and emits its Graph task, layers two and three replay the cached -Definition when their ChipTensor metadata and boundary scalar count match. Each -invocation patches the current layer's `token_position`; it is a dynamic -boundary scalar refreshed on every submission and is not part of the Graph key. - -## Definition - -Recording uses host-only C++ state: - -- `std::vector` for nodes, tensors, scalars, fanins, and pending uploads; -- `std::unordered_map` for the per-run Definition cache; -- `std::unique_ptr` for the active recording. - -The cache stores at most 16 Definitions and allocates each entry to its actual -serialized size. No fixed maximum-size recording array is copied on a cache -hit. - -At `graph_end`, recording is compacted into one contiguous, pointer-free POD -Definition. It contains: - -- node order and AIC/AIV/MIX/SPMD kernel metadata; -- `root_indices` plus both directions of the immutable topology: - fanin CSR and fanout CSR; -- one packed-heap offset per node; -- each node's ChipTensor source: - `BOUNDARY_EXACT`, `BOUNDARY_VIEW`, `INTERNAL`, or `OWN_OUTPUT`; -- fixed scalar values plus boundary-scalar source indices; -- fixed boundary signatures and alias representatives. - -The header also carries a content hash of the complete Definition image. The -device execution pool requires this hash, the Graph key, and the node count to -all match before reusing a resident Definition. A new run may record different -metadata under the same function identity, so key-only reuse is not safe. - -All references are 32-bit offsets from the Definition base. Cross-boundary -Tensors use the fixed-width `GraphTensor` wire POD rather than the -64-byte-aligned C++ `ChipTensor` object. The upload is therefore one contiguous -copy with no raw Host pointers and no relocation pass. - -Before materialization, the Scheduler recomputes the Definition content hash -and validates section ranges, topology indices, node heap offsets, the outer -heap extent, ChipTensor metadata, and ChipTensor-source bounds. Invalid wire data is -rejected before an offset participates in pointer arithmetic. - -There is no cache schema version. The cache is per run and starts empty, so a -persistent-format version would currently have no effect. - -## Cache hit and memory - -For a cache hit, the Host Orchestrator: - -1. validates the fixed boundary contract; -2. reserves one task-window slot; -3. reserves one heap block large enough for every internal intermediate; -4. computes only external fanin and boundary tensormap effects; -5. emits one outer `GRAPH` task; -6. stages the exact-size POD submission image for upload after orchestration; -7. asks the host runtime for an aligned execution block sized from the recorded - node count, Tensor-address and scalar patch capacities, and Definition - bytes, then writes that device address into the submission wire image. - -Internal nodes consume no ring task-window slots. Their descriptor, payload, -and slot state are built in host-owned GM. The runtime retains one grow-only -block per `(pipeline slot, Graph key, occurrence index)`: repeated runs on the -same slot reuse the allocation, repeated uses of one key within a run receive -distinct blocks, and the two pipeline slots never share an active block. Every -allocation goes through the Worker's tracked `MemoryAllocator`, contributes to -`committed_device_memory()`, and is released when the Worker is finalized. - -The `GraphSubmission` wire POD carries the aligned device address and usable -byte capacity explicitly. The Scheduler validates both before placement- -constructing `GraphExecution`; it never allocates execution storage from the -AICPU process heap. A block whose prior Definition key and content hash match -retains the local Definition, static node fields, and the Tensor-address and -scalar patch tables generated during its first materialization. That -graph-affine replay skips -topology binding, per-node count/offset validation, tensor-source -classification, tensor wire validation, static field stores, and static scalar -copies. It refreshes only task IDs, packed-buffer bases, boundary/internal -tensor addresses, boundary scalar bindings, scheduling state, dispatch -atomics, and wake registrations. - -The retained blocks are addressed directly by `(pipeline slot, Graph key, -occurrence index)`. Occurrence numbering restarts deterministically for every -run, so repeated layers map back to the same block in their pipeline slot; -affinity does not depend on a recycler selecting a recently freed block. - -## Scheduler flow - -Host orchestration builds the complete task image before device execution. At -the end of orchestration, the Host uploads every exact-size Graph POD image, -relocates the task and payload pointers in the shared-memory image, copies the -complete shared-memory/runtime-arena image to the device, and then launches the -resident Scheduler. - -All AICPU threads classify disjoint slices of the completed task window behind -one startup barrier. A Graph task enters preparation and external-fanin -classification during that scan, so Graph execution is interleaved with other -ready tasks at the same scheduling level once the Scheduler starts. - -This design does not overlap orchestration and scheduling within one run. -Prepared-successor pipelining can overlap preparation of run N+1 with device -execution of run N, while Graph cache hits reduce repeated orchestration work -inside a run. - -A Graph is placed in two independent control flows: - -- `graph_prepare_queue`: materialize the saved nodes even while external fanin - is still pending; -- `graph_ready_queue`: signal that the outer Graph's external fanin is ready. - -Core-owning Scheduler threads pop at most one item from each queue per loop. A -prepare call expands at most four nodes and requeues unfinished work, -interleaving Graph expansion with normal scheduling. - -Preparation and external readiness set two bits in one atomic activation gate. -Whichever operation sets the second bit activates the saved root nodes exactly -once. - -Internal dependency readiness borrows the completion-state polling idea, but -dependency wiring remains an Orchestrator responsibility: - -- recording constructs both fanin and fanout CSR in the immutable Definition; -- first materialization builds static runnable node state plus compact Tensor - address and scalar patch tables; affine replay applies those tables and - resets only dynamic runnable state; -- materialization registers each non-root on one producer selected from its - saved fanin CSR; -- a node's release/acquire `task_state` is its Graph-local completion flag, so - internal nodes need neither ring completion flags nor task-window slots; -- producer completion closes and drains only its current wake-list rather than - traversing the saved fanout CSR; -- a woken consumer scans its saved fanin CSR and either enters its shape queue - or registers on the next incomplete producer; -- `WAKE_LIST_SENTINEL` closes the completion/registration race: a failed - registration observes completion and immediately rescans. - -The runtime wake-list registration is a transient polling subscription, not -dependency discovery or Graph rewiring. Fanout CSR remains in the Definition -as part of the complete recorded topology and for DFX, but readiness does not -walk it. - -```text -outer GRAPH - -> activate root_indices[] - -> producer completion drains its current wake-list - -> each waiter polls saved fanin completion state - -> ready waiter enters its ordinary shape queue - or registers on another incomplete producer - -> final internal completion completes the outer GRAPH -``` - -Internal nodes count as zero outer ring tasks. The final node completes -the one outer Graph task, publishes the outer ring completion flag, wakes -external consumers, and contributes one to the host-visible completion count. - -Localization or materialization failure is fail-fast: the Scheduler latches an -error instead of leaving an already-submitted outer Graph unable to complete. - -## Current unsupported cases - -These cases assert in debug builds and execute through the ordinary path in a -release build: - -- an empty Graph boundary; -- variable ChipTensor shape or metadata; -- changed boundary aliasing; -- runtime-allocated boundary outputs; -- nested Graph recording; -- dispatch predicates; -- cross-boundary explicit dependencies that are not represented by a boundary - ChipTensor's creator; -- an unclassifiable internal ChipTensor source; -- a boundary-derived scalar accessed through mutable `scalar()`; -- more than 16 Definitions, 1024 internal nodes, or 32 boundary Tensors; -- insufficient task-window or heap capacity detected before outer submission. - -An AICPU execution-pool or materialization failure happens after the outer -Graph has already been submitted. It therefore latches a Scheduler fatal error -instead of falling back; leaving the outer task pending would otherwise wedge -completion. - -Explicit dependencies between recorded internal nodes are preserved when they -are otherwise supported; ordinary ChipTensor dependencies are always preserved. - -## DFX - -With L2 swimlane level 4: - -- `Graph Execution` spans an outer Graph execution; -- `AICPU Scheduler` shows bounded `graph_prepare` slices separately from normal - dispatch; -- existing Scheduler and Worker lanes show the expanded internal tasks. - -The scene coverage under `tests/st/a2a3/host_build_graph/graph_execution` -includes an AIV fanin/fanout DAG, a Qwen-style AIV/AIC decoder-layer DAG, a -three-slot multi-block MIX/SPMD Graph. Every scene invokes the same fixed Graph -three times: one recording execution followed by two outer-Graph submissions. -The full-model example at `examples/a2a3/host_build_graph/qwen3_14b_decode` -records one Qwen3-14B decoder layer and replays its Definition for the -remaining 39 layers. +Graph Execution is architecture-neutral. See the +[shared Graph Execution design](../../../../common/host_build_graph/docs/GRAPH_EXECUTION.md). diff --git a/src/a2a3/runtime/host_build_graph/runtime/graph_cache.h b/src/a2a3/runtime/host_build_graph/runtime/graph_cache.h index 8455b25b1c..e99f37bb89 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/graph_cache.h +++ b/src/a2a3/runtime/host_build_graph/runtime/graph_cache.h @@ -11,76 +11,4 @@ #pragma once -#include -#include - -#include - -#include "pto_task_id.h" -#include "pto_types.h" - -inline constexpr uint32_t GRAPH_MAX_TENSOR_ARGS = 32; - -struct GraphScopeResult { - bool execute_block{true}; - bool recording{false}; - PTO2TaskId task_id{PTO2TaskId::invalid()}; -}; - -using GraphSubmitResult = GraphScopeResult; - -constexpr uint64_t graph_hash_byte(uint64_t h, uint8_t b) { return (h ^ static_cast(b)) * 1099511628211ULL; } - -inline uint64_t graph_hash_bytes(uint64_t h, const void *data, size_t bytes) { - const auto *p = static_cast(data); - for (size_t i = 0; i < bytes; ++i) { - h = graph_hash_byte(h, p[i]); - } - return h; -} - -constexpr uint64_t graph_const_hash_impl(const char *s, uint64_t h) { - return (*s == '\0') ? h : graph_const_hash_impl(s + 1, graph_hash_byte(h, static_cast(*s))); -} - -constexpr uint64_t GRAPH_KEY(const char *s) { return graph_const_hash_impl(s, 1469598103934665603ULL); } - -inline bool rt_graph_args_cacheable(const CoreTaskArgs &args) { - if (args.has_error || args.tensor_count() <= 0 || - args.tensor_count() > static_cast(GRAPH_MAX_TENSOR_ARGS)) { - return false; - } - for (int32_t i = 0; i < args.tensor_count(); ++i) { - // A Graph boundary is caller-owned storage. Runtime-allocated - // TensorCreateInfo outputs remain on the ordinary submit path. - if (args.tag(i) == TensorArgType::OUTPUT) return false; - } - return true; -} - -inline uint64_t rt_graph_make_key(uint64_t graph_id) { return graph_id; } - -template -inline uint64_t graph_hash_config_value(uint64_t hash, T value) { - using Value = std::remove_cv_t>; - static_assert( - std::is_integral_v || std::is_same_v || std::is_same_v, - "Graph construction parameters must be integral, float, or double values" - ); - constexpr uint8_t category = std::is_same_v ? 1 : - std::is_integral_v ? (std::is_signed_v ? 2 : 3) : - 4; - constexpr uint8_t width = sizeof(Value); - hash = graph_hash_byte(hash, category); - hash = graph_hash_byte(hash, width); - return graph_hash_bytes(hash, &value, sizeof(value)); -} - -template -inline uint64_t rt_graph_make_key(uint64_t graph_id, Config... config) { - uint64_t hash = graph_hash_bytes(1469598103934665603ULL, &graph_id, sizeof(graph_id)); - const uint32_t count = sizeof...(Config); - hash = graph_hash_bytes(hash, &count, sizeof(count)); - ((hash = graph_hash_config_value(hash, config)), ...); - return hash; -} +#include "host_build_graph/graph_cache.h" diff --git a/src/a2a3/runtime/host_build_graph/runtime/graph_execution.h b/src/a2a3/runtime/host_build_graph/runtime/graph_execution.h index 9c3390f432..c2176ce4d3 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/graph_execution.h +++ b/src/a2a3/runtime/host_build_graph/runtime/graph_execution.h @@ -11,465 +11,4 @@ #pragma once -#include -#include - -#include -#include - -#include "pto_runtime2_types.h" -#include "tensor.h" - -inline constexpr uint32_t GRAPH_MAX_NODES = 1024; -inline constexpr int32_t GRAPH_MATERIALIZE_SLICE_NODES = 4; - -enum class GraphTensorSource : uint8_t { - BOUNDARY_EXACT = 0, - BOUNDARY_VIEW = 1, - INTERNAL = 2, - OWN_OUTPUT = 3, -}; - -// Wire representation of ChipTensor. ChipTensor itself is a host/runtime C++ type with -// 64-byte alignment and helper methods; placing it inside vector -// would not guarantee that alignment. Keep the boundary image C-compatible and -// copy only semantic fields into this naturally 8-byte-aligned POD. -struct GraphTensor { - uint64_t buffer_addr; - uint64_t buffer_size; - uint64_t owner_task_id; - uint64_t start_offset; - uint64_t extent_elem; - int32_t version; - uint32_t shapes[MAX_TENSOR_DIMS]; - uint32_t strides[MAX_TENSOR_DIMS]; - uint8_t ndims; - uint8_t dtype; - uint8_t manual_dep; - uint8_t is_contiguous; - uint8_t address_space; - uint8_t reserved[3]; -}; - -// Everything from GraphTensorSourceRef through GraphSubmission is copied -// across the host-device boundary. Keep it pointer-free, fixed-width and -// position-independent: every reference is an offset from its owning header. -struct GraphTensorSourceRef { - uint8_t source; - uint8_t reserved; - uint16_t source_index; - uint32_t reserved2; - uint64_t packed_offset; -}; - -enum class GraphScalarSource : uint8_t { - STATIC_VALUE = 0, - BOUNDARY = 1, -}; - -struct GraphScalarSourceRef { - uint16_t source_index; - uint8_t source; - uint8_t reserved; -}; - -struct GraphNodeDefinition { - int32_t kernel_id[PTO2_SUBTASK_SLOT_COUNT]; - uint8_t active_mask; - uint8_t task_attrs; - int16_t logical_block_num; - int16_t total_required_subtasks; - uint16_t reserved; - int32_t tensor_count; - int32_t scalar_count; - int32_t total_output_size; - uint32_t tensor_offset; - uint32_t scalar_offset; - ArgsDumpTaskMetadata dump_metadata; -}; - -struct GraphBoundarySignature { - uint64_t buffer_size; - uint32_t shapes[MAX_TENSOR_DIMS]; - uint32_t strides[MAX_TENSOR_DIMS]; - uint16_t alias_rep; - uint8_t ndims; - uint8_t dtype; - uint8_t tag; - uint8_t manual_dep; - uint8_t is_contiguous; - uint8_t reserved; -}; - -struct GraphDefinition { - uint64_t full_key; - uint64_t content_hash; - uint64_t required_heap; - uint32_t total_bytes; - uint32_t task_count; - uint32_t edge_count; - uint32_t root_count; - uint32_t boundary_count; - uint32_t boundary_scalar_count; - uint32_t tensor_arg_count; - uint32_t scalar_arg_count; - uint32_t off_fanout_offsets; - uint32_t off_fanout_indices; - uint32_t off_fanin_offsets; - uint32_t off_fanin_indices; - uint32_t off_root_indices; - uint32_t off_node_offsets; - uint32_t off_nodes; - uint32_t off_tensors; - uint32_t off_tensor_sources; - uint32_t off_scalars; - uint32_t off_scalar_sources; - uint32_t off_boundary_signatures; -}; - -struct GraphSubmission { - uint64_t graph_key; - uint64_t execution_storage; - uint64_t execution_storage_bytes; - uint64_t local_execution; - uint32_t activation_gate; - uint32_t total_bytes; - uint32_t definition_offset; - uint32_t tensors_offset; - uint32_t tensor_count; - uint32_t scalars_offset; - uint32_t scalar_count; -}; - -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); - -inline GraphTensor graph_tensor_pack(const ChipTensor &tensor) { - GraphTensor packed{}; - packed.buffer_addr = tensor.buffer.addr; - packed.buffer_size = tensor.buffer.size; - packed.owner_task_id = tensor.owner_task_id.raw; - packed.start_offset = tensor.start_offset; - packed.extent_elem = tensor.extent_elem_cache; - packed.version = tensor.version; - for (uint32_t i = 0; i < tensor.ndims; ++i) { - packed.shapes[i] = tensor.shapes[i]; - packed.strides[i] = tensor.strides[i]; - } - packed.ndims = static_cast(tensor.ndims); - packed.dtype = static_cast(tensor.dtype); - packed.manual_dep = tensor.manual_dep ? 1 : 0; - packed.is_contiguous = tensor.is_contiguous ? 1 : 0; - packed.address_space = static_cast(tensor.address_space); - return packed; -} - -inline void graph_tensor_unpack(const GraphTensor &packed, ChipTensor *tensor) { - tensor->buffer = PTOBufferHandle{packed.buffer_addr, packed.buffer_size}; - tensor->owner_task_id = PTO2TaskId{packed.owner_task_id}; - tensor->start_offset = packed.start_offset; - tensor->extent_elem_cache = packed.extent_elem; - tensor->version = packed.version; - tensor->ndims = packed.ndims; - tensor->dtype = static_cast(packed.dtype); - tensor->manual_dep = packed.manual_dep != 0; - tensor->is_contiguous = packed.is_contiguous != 0; - tensor->address_space = static_cast(packed.address_space); - for (uint32_t i = 0; i < MAX_TENSOR_DIMS; ++i) { - tensor->shapes[i] = packed.shapes[i]; - tensor->strides[i] = packed.strides[i]; - } - for (uint8_t &byte : tensor->_pad_cl2) - byte = 0; -} - -inline bool graph_tensor_wire_valid(const GraphTensor &tensor) { - if (tensor.buffer_addr == 0 || tensor.ndims == 0 || tensor.ndims > MAX_TENSOR_DIMS || - tensor.dtype >= static_cast(DataType::DATA_TYPE_NUM) || tensor.manual_dep > 1 || - tensor.is_contiguous > 1 || tensor.address_space > 1) { - return false; - } - - uint64_t extent = 1; - uint64_t expected_stride = 1; - bool contiguous = true; - for (int32_t i = static_cast(tensor.ndims) - 1; i >= 0; --i) { - const uint64_t shape = tensor.shapes[i]; - const uint64_t stride = tensor.strides[i]; - if (shape == 0 || stride == 0) return false; - contiguous &= stride == expected_stride; - if (shape - 1 > (UINT64_MAX - extent) / stride || expected_stride > UINT64_MAX / shape) return false; - extent += (shape - 1) * stride; - expected_stride *= shape; - } - if (extent != tensor.extent_elem || contiguous != (tensor.is_contiguous != 0)) return false; - - const uint64_t element_size = get_element_size(static_cast(tensor.dtype)); - const uint64_t buffer_elements = tensor.buffer_size / element_size; - return tensor.start_offset <= buffer_elements && tensor.extent_elem <= buffer_elements - tensor.start_offset; -} - -template -inline const T *graph_definition_array(const GraphDefinition &definition, uint32_t offset, uint32_t count) { - if (offset == 0 || offset > definition.total_bytes || offset % alignof(T) != 0) return nullptr; - const size_t remaining = static_cast(definition.total_bytes - offset); - if (count > remaining / sizeof(T)) return nullptr; - return reinterpret_cast(reinterpret_cast(&definition) + offset); -} - -template -inline const T *graph_definition_ptr(const GraphDefinition &definition, uint32_t offset) { - return graph_definition_array(definition, offset, 1); -} - -inline GraphSubmission *graph_submission_from_slot(PTO2TaskSlotState &slot) { - return slot.task_kind == TaskKind::GRAPH ? static_cast(slot.graph_context) : nullptr; -} - -inline const GraphDefinition *graph_submission_definition(const GraphSubmission &submission) { - if (submission.definition_offset == 0 || submission.definition_offset % alignof(GraphDefinition) != 0 || - submission.definition_offset > submission.total_bytes || - sizeof(GraphDefinition) > submission.total_bytes - submission.definition_offset) { - return nullptr; - } - const auto *definition = reinterpret_cast( - reinterpret_cast(&submission) + submission.definition_offset - ); - if (definition->total_bytes < sizeof(GraphDefinition) || - definition->total_bytes > submission.total_bytes - submission.definition_offset) { - return nullptr; - } - return definition; -} - -inline bool graph_submission_wire_size_valid(const GraphSubmission &submission, size_t available_bytes) { - return available_bytes >= sizeof(GraphSubmission) && submission.total_bytes == available_bytes; -} - -inline const GraphTensor *graph_submission_tensors(const GraphSubmission &submission) { - if (submission.tensors_offset == 0 || submission.tensors_offset % alignof(GraphTensor) != 0 || - submission.tensors_offset > submission.total_bytes || - submission.tensor_count > (submission.total_bytes - submission.tensors_offset) / sizeof(GraphTensor)) { - return nullptr; - } - return reinterpret_cast( - reinterpret_cast(&submission) + submission.tensors_offset - ); -} - -inline const uint64_t *graph_submission_scalars(const GraphSubmission &submission) { - if (submission.scalar_count == 0) return nullptr; - if (submission.scalars_offset == 0 || submission.scalars_offset % alignof(uint64_t) != 0 || - submission.scalars_offset > submission.total_bytes || - submission.scalar_count > (submission.total_bytes - submission.scalars_offset) / sizeof(uint64_t)) { - return nullptr; - } - return reinterpret_cast( - reinterpret_cast(&submission) + submission.scalars_offset - ); -} - -enum class GraphExecutionState : uint8_t { - SUBMITTED = 0, - MATERIALIZING = 1, - PREPARED = 2, - ACTIVE = 3, - COMPLETED = 4, -}; - -enum class GraphMaterializeResult : uint8_t { - INVALID = 0, - BUSY = 1, - PENDING = 2, - PREPARED = 3, -}; - -enum class GraphTensorAddressSource : uint8_t { - BOUNDARY = 0, - INTERNAL = 1, -}; - -// Precomputed on the first materialization and retained next to the node -// storage. Affine replay walks this compact POD instead of re-reading and -// classifying GraphTensorSourceRef entries from the Definition. -struct GraphTensorAddressPatch { - uint64_t address_offset; - uint16_t source_index; - uint8_t source; - uint8_t reserved[5]; -}; - -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(sizeof(GraphTensorAddressPatch) == 16); - -struct GraphScalarPatch { - uint16_t node_index; - uint8_t node_scalar_index; - uint8_t boundary_scalar_index; -}; - -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(sizeof(GraphScalarPatch) == 4); -static_assert(GRAPH_MAX_NODES <= UINT16_MAX); -static_assert(MAX_SCALAR_ARGS <= UINT8_MAX); - -struct alignas(64) GraphNodeStorage { - PTO2TaskDescriptor task; - PTO2TaskPayload payload; - PTO2TaskSlotState slot; -}; - -inline constexpr uint64_t GRAPH_EXECUTION_STORAGE_MAGIC = 0x4752415048455845ULL; -inline constexpr uint64_t GRAPH_EXECUTION_INITIALIZING = 1; - -struct GraphExecution { - uint64_t storage_magic{0}; - std::atomic state{GraphExecutionState::SUBMITTED}; - std::atomic materialize_busy{0}; - std::atomic remaining_nodes{0}; - std::atomic retired_nodes{0}; - // Incremental activation: nodes in [0, published_nodes) are fully - // materialized and registered, so a route pass may consider them. route_cursor - // is the next such node index a route pass will claim; roots below it have - // been pushed to the ready queue exactly once. Both advance monotonically and - // reset per (re)submission. - std::atomic published_nodes{0}; - std::atomic route_cursor{0}; - int32_t node_count{0}; - int32_t node_capacity{0}; - int32_t materialized_nodes{0}; - int32_t materialized_node_count{0}; - int32_t constructed_nodes{0}; - uint32_t tensor_patch_capacity{0}; - uint32_t materialized_tensor_patches{0}; - uint32_t materialized_tensor_patch_count{0}; - uint32_t scalar_patch_capacity{0}; - uint32_t materialized_scalar_patches{0}; - uint32_t materialized_scalar_patch_count{0}; - size_t allocation_bytes{0}; - size_t definition_capacity{0}; - uint64_t graph_key{0}; - uint64_t definition_hash{0}; - uint64_t materialized_graph_key{0}; - uint64_t materialized_definition_hash{0}; - uintptr_t materialized_outer_base{0}; - bool definition_affine_reuse{false}; - PTO2TaskSlotState *outer_slot{nullptr}; - GraphNodeStorage *nodes{nullptr}; - GraphNodeStorage *node_storage{nullptr}; - GraphTensorAddressPatch *tensor_patches{nullptr}; - GraphScalarPatch *scalar_patches{nullptr}; - void *definition_storage{nullptr}; - const GraphDefinition *definition{nullptr}; - const uint32_t *fanin_offsets{nullptr}; - const uint16_t *fanin_indices{nullptr}; - const GraphTensor *boundary_tensors{nullptr}; - uint32_t boundary_tensor_count{0}; - const uint64_t *boundary_scalars{nullptr}; - uint32_t boundary_scalar_count{0}; -}; - -static_assert(offsetof(GraphExecution, storage_magic) == 0); -static_assert(sizeof(GraphExecution::storage_magic) == sizeof(uint64_t)); -static_assert(std::is_trivially_destructible_v); -static_assert(std::is_trivially_destructible_v); - -inline bool graph_execution_storage_layout( - int32_t node_capacity, uint32_t tensor_patch_capacity, uint32_t scalar_patch_capacity, size_t definition_capacity, - size_t *nodes_offset, size_t *tensor_patches_offset, size_t *scalar_patches_offset, size_t *definition_offset, - size_t *storage_bytes -) { - if (nodes_offset == nullptr || tensor_patches_offset == nullptr || scalar_patches_offset == nullptr || - definition_offset == nullptr || storage_bytes == nullptr || node_capacity <= 0 || - static_cast(node_capacity) > SIZE_MAX / sizeof(GraphNodeStorage) || - tensor_patch_capacity > GRAPH_MAX_NODES * MAX_TENSOR_ARGS || - scalar_patch_capacity > GRAPH_MAX_NODES * MAX_SCALAR_ARGS) { - return false; - } - auto checked_align_up = [](size_t value, size_t alignment, size_t *result) { - if (alignment == 0 || value > SIZE_MAX - (alignment - 1)) return false; - *result = (value + alignment - 1) & ~(alignment - 1); - return true; - }; - const size_t nodes_bytes = static_cast(node_capacity) * sizeof(GraphNodeStorage); - const size_t tensor_patches_bytes = static_cast(tensor_patch_capacity) * sizeof(GraphTensorAddressPatch); - const size_t scalar_patches_bytes = static_cast(scalar_patch_capacity) * sizeof(GraphScalarPatch); - if (!checked_align_up(sizeof(GraphExecution), alignof(GraphNodeStorage), nodes_offset) || - *nodes_offset > SIZE_MAX - nodes_bytes || - !checked_align_up(*nodes_offset + nodes_bytes, alignof(GraphTensorAddressPatch), tensor_patches_offset) || - *tensor_patches_offset > SIZE_MAX - tensor_patches_bytes || - !checked_align_up( - *tensor_patches_offset + tensor_patches_bytes, alignof(GraphScalarPatch), scalar_patches_offset - ) || - *scalar_patches_offset > SIZE_MAX - scalar_patches_bytes || - !checked_align_up(*scalar_patches_offset + scalar_patches_bytes, alignof(GraphDefinition), definition_offset) || - *definition_offset > SIZE_MAX - definition_capacity) { - return false; - } - return checked_align_up(*definition_offset + definition_capacity, alignof(GraphNodeStorage), storage_bytes); -} - -inline bool graph_execution_storage_bytes( - int32_t node_capacity, uint32_t tensor_patch_capacity, uint32_t scalar_patch_capacity, size_t definition_capacity, - size_t *storage_bytes -) { - size_t nodes_offset = 0; - size_t tensor_patches_offset = 0; - size_t scalar_patches_offset = 0; - size_t definition_offset = 0; - return graph_execution_storage_layout( - node_capacity, tensor_patch_capacity, scalar_patch_capacity, definition_capacity, &nodes_offset, - &tensor_patches_offset, &scalar_patches_offset, &definition_offset, storage_bytes - ); -} - -GraphExecution *graph_execution_localize(PTO2TaskSlotState &outer_slot); -GraphMaterializeResult graph_execution_materialize_slice( - PTO2TaskSlotState &outer_slot, GraphExecution &execution, int32_t max_nodes, int32_t *nodes_materialized = nullptr -); - -inline GraphExecution *graph_execution_from_slot(PTO2TaskSlotState &slot) { - return slot.task_kind == TaskKind::GRAPH_NODE ? static_cast(slot.graph_context) : nullptr; -} - -inline bool graph_execution_complete_node(GraphExecution &execution) { - return execution.remaining_nodes.fetch_sub(1, std::memory_order_acq_rel) == 1; -} - -inline void graph_execution_mark_completed(GraphExecution &execution) { - execution.state.store(GraphExecutionState::COMPLETED, std::memory_order_release); -} - -inline void graph_execution_retire_node(GraphExecution &execution) { - execution.retired_nodes.fetch_add(1, std::memory_order_release); -} - -inline bool graph_submission_signal(GraphSubmission &submission, uint32_t bit) { - constexpr uint32_t BOTH = 0x3; - uint32_t observed = __atomic_fetch_or(&submission.activation_gate, bit, __ATOMIC_ACQ_REL); - return observed != BOTH && (observed | bit) == BOTH; -} - -inline GraphExecution *graph_submission_local_execution(GraphSubmission &submission) { - uint64_t raw = __atomic_load_n(&submission.local_execution, __ATOMIC_ACQUIRE); - if (raw <= GRAPH_EXECUTION_INITIALIZING) return nullptr; - return reinterpret_cast(static_cast(raw)); -} - -inline bool graph_submission_execution_initializing(const GraphSubmission &submission) { - return __atomic_load_n(&submission.local_execution, __ATOMIC_ACQUIRE) == GRAPH_EXECUTION_INITIALIZING; -} +#include "host_build_graph/graph_execution.h" diff --git a/src/a2a3/runtime/host_build_graph/runtime/graph_host_state.h b/src/a2a3/runtime/host_build_graph/runtime/graph_host_state.h index 4d869b1c4d..885c97660e 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/graph_host_state.h +++ b/src/a2a3/runtime/host_build_graph/runtime/graph_host_state.h @@ -11,27 +11,4 @@ #pragma once -#include -#include -#include - -struct PTO2TaskSlotState; -struct GraphHostState; - -inline constexpr size_t GRAPH_MAX_DEFINITIONS = 16; - -struct GraphHostStateDeleter { - void operator()(GraphHostState *state) const noexcept; -}; - -using GraphHostStatePtr = std::unique_ptr; - -struct GraphHostUpload { - PTO2TaskSlotState *outer_slot; - std::byte *data; - size_t bytes; -}; - -GraphHostStatePtr make_graph_host_state(); -size_t graph_host_upload_count(const GraphHostState &state); -std::optional graph_host_upload(GraphHostState &state, size_t index); +#include "host_build_graph/graph_host_state.h" diff --git a/src/a5/runtime/host_build_graph/build_config.py b/src/a5/runtime/host_build_graph/build_config.py index 1a9cfd5ca7..35d5036ec5 100644 --- a/src/a5/runtime/host_build_graph/build_config.py +++ b/src/a5/runtime/host_build_graph/build_config.py @@ -24,7 +24,10 @@ BUILD_CONFIG = { "aicore": {"include_dirs": ["runtime", "common", ".."], "source_dirs": ["aicore", "orchestration"]}, - "aicpu": {"include_dirs": ["runtime", "common", ".."], "source_dirs": ["aicpu", "runtime", "orchestration"]}, + "aicpu": { + "include_dirs": ["runtime", "common", ".."], + "source_dirs": ["aicpu", "runtime", "orchestration", "../../../common/host_build_graph"], + }, "host": { "include_dirs": ["runtime", "common", ".."], "source_dirs": ["host", "runtime/orchestrator_core", "runtime/shared", "orchestration"], diff --git a/src/a5/runtime/host_build_graph/docs/GRAPH_EXECUTION.md b/src/a5/runtime/host_build_graph/docs/GRAPH_EXECUTION.md index 9d854d1aea..d757e463a4 100644 --- a/src/a5/runtime/host_build_graph/docs/GRAPH_EXECUTION.md +++ b/src/a5/runtime/host_build_graph/docs/GRAPH_EXECUTION.md @@ -1,384 +1,4 @@ # Graph Execution -Graph Execution is available only in the `host_build_graph` runtime. A Graph is -a composite incore task: it is submitted and completed once like an AIC, AIV, -MIX, or SPMD task, but contains a recorded task DAG. - -Every invocation places exactly one `GRAPH` task in the host task window. The -first invocation records the DAG off the ring — its internal submissions build -host-only node metadata and reserve scratch output buffers instead of consuming -task-window slots — then emits the outer `GRAPH` task from the freshly built -Definition. Later invocations reuse the cached Definition and emit the same one -`GRAPH` task directly. In both cases the device Scheduler expands the saved -topology and dispatches the internal nodes; the Host Orchestrator never submits -those nodes as ring tasks. - -A recording that hits an unsupported construct is discarded and the body re-runs -on the ordinary task-submit path so its work is still submitted; the internal -nodes then occupy the ring only for that one fallback invocation. - -## API - -A Graph uses `CoreTaskArgs`, the existing incore argument type: - -```cpp -void graph_function(const CoreTaskArgs &args, int variant) { - const ChipTensor &input = args.tensor(0).ref(); - const ChipTensor &weight = args.tensor(1).ref(); - const ChipTensor &output = args.tensor(2).ref(); - - const std::array shape{input.shapes[0]}; - TensorCreateInfo intermediate( - shape.data(), static_cast(shape.size()), input.dtype - ); - - CoreTaskArgs matmul_args; - matmul_args.add_input(input, weight); - matmul_args.add_output(intermediate); - matmul_args.copy_scalars_from(args, 0, 1); // current invocation's value - TaskOutputTensors matmul = rt_submit_aic_task( - variant == 0 ? FUNC_MATMUL : FUNC_MATMUL_TRANSPOSED, - matmul_args - ); - - CoreTaskArgs activation_args; - activation_args.add_input(matmul.get_ref(0)); - activation_args.add_output(output); - rt_submit_aiv_task(FUNC_ACTIVATION, activation_args); -} - -void submit_layer(const CoreTaskArgs &args) { - rt_submit_graph(&graph_function, args, /*variant=*/0); -} -``` - -The function pointer is the default Graph identity. Trailing integral, -`float`, `double`, and `bool` construction parameters are forwarded to the -Graph function and hashed by value into the cache key. They are separate from -execution scalars in `CoreTaskArgs`: changing a construction parameter selects a -different Definition rather than patching an existing one. - -An explicit identity is available for call sites that need a stable name: - -```cpp -rt_submit_graph( - GRAPH_KEY("qwen_decoder_layer_v1"), - &graph_function, - args, - /*variant=*/0 -); -``` - -An explicit `GRAPH_KEY` must be unique for every distinct Graph function in an -orchestration callable. The explicit-key overload deliberately excludes the -Graph function pointer from the cache identity so the key remains stable; using -the same key for different functions can select the wrong recorded topology. - -There are no public `GraphArgs`, `GraphBindings`, `Patch`, or `ScalarRef` -types. The boundary is represented by `CoreTaskArgs`. - -Boundary scalars are pass-through bindings. Forward them directly with -`node_args.add_scalar(args.scalar(i))` or `copy_scalars_from(args, i, count)` -so recording can retain their source indices. - -Ordinary C++ value transformations do not retain boundary provenance. Both -`node_args.add_scalar(args.scalar(i) + 1)` and copying `args.scalar(i)` into a -local arithmetic variable before calling `add_scalar` produce an ordinary -static node scalar. That value is stored in the Definition, and later cache -hits reuse the first invocation's value without a warning. The runtime cannot -distinguish such a derived value from an intentional static literal after the -C++ expression has produced a plain arithmetic value. Compute the derived value -before constructing the Graph boundary and pass it as another boundary scalar, -perform the transformation in a kernel, or use a construction parameter when -the value changes the Graph structure. - -Access through a non-const `scalar()` invalidates inherited boundary provenance -conservatively, because returning a mutable reference cannot distinguish a read -from a later write. A Graph containing such an invalidated binding is not -cached, which prevents replay from silently replacing the transformed value -with the unmodified boundary value. - -## Supported dynamic and static data - -- Boundary ChipTensor addresses may change for every invocation. -- Boundary scalar values may change for every invocation. Their count is fixed - by the recorded boundary contract. Unused boundary scalars are allowed and do - not create internal scalar patches. -- A Graph boundary contains at least one ChipTensor. -- Construction parameters are part of Graph identity and may control the - function's task count, kernel selection, or other structural choices. -- Boundary ChipTensor shape, stride, dtype, size, direction, contiguity, and alias - partition must match the first invocation. -- Internal task scalars with no boundary source are fixed Definition data. -- Boundary storage is caller-owned. `INPUT`, `INOUT`, `OUTPUT_EXISTING`, and - `NO_DEP` are supported. A boundary `TensorCreateInfo` tagged `OUTPUT` is not. -- Early-resolve hints apply while recording the first invocation. Replayed - internal nodes use the saved completion topology without the hint. -- A recorded task may depend on a Graph-external producer when that producer - is the creator of a boundary ChipTensor. The outer Graph owns that dependency on - replay; arbitrary cross-boundary explicit dependencies remain unsupported. - -Structural or alias mismatch logs a warning and executes the Graph function -normally for that invocation. It never reuses heap offsets recorded for a -different shape. Debug builds also assert at these unsupported boundaries so -development catches a violated fixed-shape contract immediately; the ordinary -path remains the defensive release-build behavior. - -## Qwen decoder-layer example - -The upper layer packages all ChipTensor I/O in `CoreTaskArgs`; the wrapper has no -separate `hidden`, `weight`, or `output` parameters: - -```cpp -void qwen_decoder_layer(const CoreTaskArgs &args) { - const ChipTensor &hidden = args.tensor(0).ref(); - const ChipTensor &attention_weight = args.tensor(1).ref(); - const ChipTensor &mlp_weight = args.tensor(2).ref(); - const ChipTensor &output = args.tensor(3).ref(); - - const std::array hidden_shape{hidden.shapes[0]}; - TensorCreateInfo attention_out( - hidden_shape.data(), static_cast(hidden_shape.size()), hidden.dtype - ); - - CoreTaskArgs attention_args; - attention_args.add_input(hidden, attention_weight); - attention_args.add_output(attention_out); - attention_args.copy_scalars_from(args, 0, 1); // dynamic token position - TaskOutputTensors attention = - rt_submit_aic_task(FUNC_ATTENTION, attention_args); - - MixedKernels mlp; - mlp.aic_kernel_id = FUNC_MLP_AIC; - mlp.aiv0_kernel_id = FUNC_MLP_AIV; - - CoreTaskArgs mlp_args; - mlp_args.add_input(attention.get_ref(0), mlp_weight); - mlp_args.add_output(output); - rt_submit_task(mlp, mlp_args); -} - -void submit_qwen_decoder_layer(const CoreTaskArgs &args) { - rt_submit_graph(&qwen_decoder_layer, args); -} - -void decode_three_layers( - const std::array &hidden, - const std::array &attention_weight, - const std::array &mlp_weight, - const std::array &output, - const std::array &token_position -) { - for (std::size_t layer = 0; layer < hidden.size(); ++layer) { - CoreTaskArgs args; - args.add_input( - hidden[layer], - attention_weight[layer], - mlp_weight[layer] - ); - args.add_output(output[layer]); - args.add_scalar(token_position[layer]); - submit_qwen_decoder_layer(args); - } -} -``` - -All three layers submit one Graph task each: the first records the sub-DAG off -the ring and emits its Graph task, layers two and three replay the cached -Definition when their ChipTensor metadata and boundary scalar count match. Each -invocation patches the current layer's `token_position`; it is a dynamic -boundary scalar refreshed on every submission and is not part of the Graph key. - -## Definition - -Recording uses host-only C++ state: - -- `std::vector` for nodes, tensors, scalars, fanins, and pending uploads; -- `std::unordered_map` for the per-run Definition cache; -- `std::unique_ptr` for the active recording. - -The cache stores at most 16 Definitions and allocates each entry to its actual -serialized size. No fixed maximum-size recording array is copied on a cache -hit. - -At `graph_end`, recording is compacted into one contiguous, pointer-free POD -Definition. It contains: - -- node order and AIC/AIV/MIX/SPMD kernel metadata; -- `root_indices` plus both directions of the immutable topology: - fanin CSR and fanout CSR; -- one packed-heap offset per node; -- each node's ChipTensor source: - `BOUNDARY_EXACT`, `BOUNDARY_VIEW`, `INTERNAL`, or `OWN_OUTPUT`; -- fixed scalar values plus boundary-scalar source indices; -- fixed boundary signatures and alias representatives. - -The header also carries a content hash of the complete Definition image. The -device execution pool requires this hash, the Graph key, and the node count to -all match before reusing a resident Definition. A new run may record different -metadata under the same function identity, so key-only reuse is not safe. - -All references are 32-bit offsets from the Definition base. Cross-boundary -Tensors use the fixed-width `GraphTensor` wire POD rather than the -64-byte-aligned C++ `ChipTensor` object. The upload is therefore one contiguous -copy with no raw Host pointers and no relocation pass. - -Before materialization, the Scheduler recomputes the Definition content hash -and validates section ranges, topology indices, node heap offsets, the outer -heap extent, ChipTensor metadata, and ChipTensor-source bounds. Invalid wire data is -rejected before an offset participates in pointer arithmetic. - -There is no cache schema version. The cache is per run and starts empty, so a -persistent-format version would currently have no effect. - -## Cache hit and memory - -For a cache hit, the Host Orchestrator: - -1. validates the fixed boundary contract; -2. reserves one task-window slot; -3. reserves one heap block large enough for every internal intermediate; -4. computes only external fanin and boundary tensormap effects; -5. emits one outer `GRAPH` task; -6. stages the exact-size POD submission image for upload after orchestration; -7. asks the host runtime for an aligned execution block sized from the recorded - node count, Tensor-address and scalar patch capacities, and Definition - bytes, then writes that device address into the submission wire image. - -Internal nodes consume no ring task-window slots. Their descriptor, payload, -and slot state are built in host-owned GM. The runtime retains one grow-only -block per `(pipeline slot, Graph key, occurrence index)`: repeated runs on the -same slot reuse the allocation, repeated uses of one key within a run receive -distinct blocks, and the two pipeline slots never share an active block. Every -allocation goes through the Worker's tracked `MemoryAllocator`, contributes to -`committed_device_memory()`, and is released when the Worker is finalized. - -The `GraphSubmission` wire POD carries the aligned device address and usable -byte capacity explicitly. The Scheduler validates both before placement- -constructing `GraphExecution`; it never allocates execution storage from the -AICPU process heap. A block whose prior Definition key and content hash match -retains the local Definition, static node fields, and the Tensor-address and -scalar patch tables generated during its first materialization. That -graph-affine replay skips -topology binding, per-node count/offset validation, tensor-source -classification, tensor wire validation, static field stores, and static scalar -copies. It refreshes only task IDs, packed-buffer bases, boundary/internal -tensor addresses, boundary scalar bindings, scheduling state, dispatch -atomics, and wake registrations. - -The retained blocks are addressed directly by `(pipeline slot, Graph key, -occurrence index)`. Occurrence numbering restarts deterministically for every -run, so repeated layers map back to the same block in their pipeline slot; -affinity does not depend on a recycler selecting a recently freed block. - -## Scheduler flow - -Host orchestration builds the complete task image before device execution. At -the end of orchestration, the Host uploads every exact-size Graph POD image, -relocates the task and payload pointers in the shared-memory image, copies the -complete shared-memory/runtime-arena image to the device, and then launches the -resident Scheduler. - -All AICPU threads classify disjoint slices of the completed task window behind -one startup barrier. A Graph task enters preparation and external-fanin -classification during that scan, so Graph execution is interleaved with other -ready tasks at the same scheduling level once the Scheduler starts. - -This design does not overlap orchestration and scheduling within one run. -Prepared-successor pipelining can overlap preparation of run N+1 with device -execution of run N, while Graph cache hits reduce repeated orchestration work -inside a run. - -A Graph is placed in two independent control flows: - -- `graph_prepare_queue`: materialize the saved nodes even while external fanin - is still pending; -- `graph_ready_queue`: signal that the outer Graph's external fanin is ready. - -Core-owning Scheduler threads pop at most one item from each queue per loop. A -prepare call expands at most four nodes and requeues unfinished work, -interleaving Graph expansion with normal scheduling. - -Preparation and external readiness set two bits in one atomic activation gate. -Whichever operation sets the second bit activates the saved root nodes exactly -once. - -Internal dependency readiness borrows the completion-state polling idea, but -dependency wiring remains an Orchestrator responsibility: - -- recording constructs both fanin and fanout CSR in the immutable Definition; -- first materialization builds static runnable node state plus compact Tensor - address and scalar patch tables; affine replay applies those tables and - resets only dynamic runnable state; -- materialization registers each non-root on one producer selected from its - saved fanin CSR; -- a node's release/acquire `task_state` is its Graph-local completion flag, so - internal nodes need neither ring completion flags nor task-window slots; -- producer completion closes and drains only its current wake-list rather than - traversing the saved fanout CSR; -- a woken consumer scans its saved fanin CSR and either enters its shape queue - or registers on the next incomplete producer; -- `WAKE_LIST_SENTINEL` closes the completion/registration race: a failed - registration observes completion and immediately rescans. - -The runtime wake-list registration is a transient polling subscription, not -dependency discovery or Graph rewiring. Fanout CSR remains in the Definition -as part of the complete recorded topology and for DFX, but readiness does not -walk it. - -```text -outer GRAPH - -> activate root_indices[] - -> producer completion drains its current wake-list - -> each waiter polls saved fanin completion state - -> ready waiter enters its ordinary shape queue - or registers on another incomplete producer - -> final internal completion completes the outer GRAPH -``` - -Internal nodes count as zero outer ring tasks. The final node completes -the one outer Graph task, publishes the outer ring completion flag, wakes -external consumers, and contributes one to the host-visible completion count. - -Localization or materialization failure is fail-fast: the Scheduler latches an -error instead of leaving an already-submitted outer Graph unable to complete. - -## Current unsupported cases - -These cases assert in debug builds and execute through the ordinary path in a -release build: - -- an empty Graph boundary; -- variable ChipTensor shape or metadata; -- changed boundary aliasing; -- runtime-allocated boundary outputs; -- nested Graph recording; -- dispatch predicates; -- cross-boundary explicit dependencies that are not represented by a boundary - ChipTensor's creator; -- an unclassifiable internal ChipTensor source; -- a boundary-derived scalar accessed through mutable `scalar()`; -- more than 16 Definitions, 1024 internal nodes, or 32 boundary Tensors; -- insufficient task-window or heap capacity detected before outer submission. - -An AICPU execution-pool or materialization failure happens after the outer -Graph has already been submitted. It therefore latches a Scheduler fatal error -instead of falling back; leaving the outer task pending would otherwise wedge -completion. - -Explicit dependencies between recorded internal nodes are preserved when they -are otherwise supported; ordinary ChipTensor dependencies are always preserved. - -## DFX - -With L2 swimlane level 4: - -- `Graph Execution` spans an outer Graph execution; -- `AICPU Scheduler` shows bounded `graph_prepare` slices separately from normal - dispatch; -- existing Scheduler and Worker lanes show the expanded internal tasks. - -The scene coverage under `tests/st/a5/host_build_graph/graph_execution` -includes an AIV fanin/fanout DAG, an AIC/AIV fanout/fanin DAG, and a three-slot -multi-block MIX/SPMD Graph. Every scene invokes the same fixed Graph three -times: one recording execution followed by two outer-Graph submissions. +Graph Execution is architecture-neutral. See the +[shared Graph Execution design](../../../../common/host_build_graph/docs/GRAPH_EXECUTION.md). diff --git a/src/a5/runtime/host_build_graph/runtime/graph_cache.h b/src/a5/runtime/host_build_graph/runtime/graph_cache.h index 8455b25b1c..e99f37bb89 100644 --- a/src/a5/runtime/host_build_graph/runtime/graph_cache.h +++ b/src/a5/runtime/host_build_graph/runtime/graph_cache.h @@ -11,76 +11,4 @@ #pragma once -#include -#include - -#include - -#include "pto_task_id.h" -#include "pto_types.h" - -inline constexpr uint32_t GRAPH_MAX_TENSOR_ARGS = 32; - -struct GraphScopeResult { - bool execute_block{true}; - bool recording{false}; - PTO2TaskId task_id{PTO2TaskId::invalid()}; -}; - -using GraphSubmitResult = GraphScopeResult; - -constexpr uint64_t graph_hash_byte(uint64_t h, uint8_t b) { return (h ^ static_cast(b)) * 1099511628211ULL; } - -inline uint64_t graph_hash_bytes(uint64_t h, const void *data, size_t bytes) { - const auto *p = static_cast(data); - for (size_t i = 0; i < bytes; ++i) { - h = graph_hash_byte(h, p[i]); - } - return h; -} - -constexpr uint64_t graph_const_hash_impl(const char *s, uint64_t h) { - return (*s == '\0') ? h : graph_const_hash_impl(s + 1, graph_hash_byte(h, static_cast(*s))); -} - -constexpr uint64_t GRAPH_KEY(const char *s) { return graph_const_hash_impl(s, 1469598103934665603ULL); } - -inline bool rt_graph_args_cacheable(const CoreTaskArgs &args) { - if (args.has_error || args.tensor_count() <= 0 || - args.tensor_count() > static_cast(GRAPH_MAX_TENSOR_ARGS)) { - return false; - } - for (int32_t i = 0; i < args.tensor_count(); ++i) { - // A Graph boundary is caller-owned storage. Runtime-allocated - // TensorCreateInfo outputs remain on the ordinary submit path. - if (args.tag(i) == TensorArgType::OUTPUT) return false; - } - return true; -} - -inline uint64_t rt_graph_make_key(uint64_t graph_id) { return graph_id; } - -template -inline uint64_t graph_hash_config_value(uint64_t hash, T value) { - using Value = std::remove_cv_t>; - static_assert( - std::is_integral_v || std::is_same_v || std::is_same_v, - "Graph construction parameters must be integral, float, or double values" - ); - constexpr uint8_t category = std::is_same_v ? 1 : - std::is_integral_v ? (std::is_signed_v ? 2 : 3) : - 4; - constexpr uint8_t width = sizeof(Value); - hash = graph_hash_byte(hash, category); - hash = graph_hash_byte(hash, width); - return graph_hash_bytes(hash, &value, sizeof(value)); -} - -template -inline uint64_t rt_graph_make_key(uint64_t graph_id, Config... config) { - uint64_t hash = graph_hash_bytes(1469598103934665603ULL, &graph_id, sizeof(graph_id)); - const uint32_t count = sizeof...(Config); - hash = graph_hash_bytes(hash, &count, sizeof(count)); - ((hash = graph_hash_config_value(hash, config)), ...); - return hash; -} +#include "host_build_graph/graph_cache.h" diff --git a/src/a5/runtime/host_build_graph/runtime/graph_execution.h b/src/a5/runtime/host_build_graph/runtime/graph_execution.h index 9c3390f432..c2176ce4d3 100644 --- a/src/a5/runtime/host_build_graph/runtime/graph_execution.h +++ b/src/a5/runtime/host_build_graph/runtime/graph_execution.h @@ -11,465 +11,4 @@ #pragma once -#include -#include - -#include -#include - -#include "pto_runtime2_types.h" -#include "tensor.h" - -inline constexpr uint32_t GRAPH_MAX_NODES = 1024; -inline constexpr int32_t GRAPH_MATERIALIZE_SLICE_NODES = 4; - -enum class GraphTensorSource : uint8_t { - BOUNDARY_EXACT = 0, - BOUNDARY_VIEW = 1, - INTERNAL = 2, - OWN_OUTPUT = 3, -}; - -// Wire representation of ChipTensor. ChipTensor itself is a host/runtime C++ type with -// 64-byte alignment and helper methods; placing it inside vector -// would not guarantee that alignment. Keep the boundary image C-compatible and -// copy only semantic fields into this naturally 8-byte-aligned POD. -struct GraphTensor { - uint64_t buffer_addr; - uint64_t buffer_size; - uint64_t owner_task_id; - uint64_t start_offset; - uint64_t extent_elem; - int32_t version; - uint32_t shapes[MAX_TENSOR_DIMS]; - uint32_t strides[MAX_TENSOR_DIMS]; - uint8_t ndims; - uint8_t dtype; - uint8_t manual_dep; - uint8_t is_contiguous; - uint8_t address_space; - uint8_t reserved[3]; -}; - -// Everything from GraphTensorSourceRef through GraphSubmission is copied -// across the host-device boundary. Keep it pointer-free, fixed-width and -// position-independent: every reference is an offset from its owning header. -struct GraphTensorSourceRef { - uint8_t source; - uint8_t reserved; - uint16_t source_index; - uint32_t reserved2; - uint64_t packed_offset; -}; - -enum class GraphScalarSource : uint8_t { - STATIC_VALUE = 0, - BOUNDARY = 1, -}; - -struct GraphScalarSourceRef { - uint16_t source_index; - uint8_t source; - uint8_t reserved; -}; - -struct GraphNodeDefinition { - int32_t kernel_id[PTO2_SUBTASK_SLOT_COUNT]; - uint8_t active_mask; - uint8_t task_attrs; - int16_t logical_block_num; - int16_t total_required_subtasks; - uint16_t reserved; - int32_t tensor_count; - int32_t scalar_count; - int32_t total_output_size; - uint32_t tensor_offset; - uint32_t scalar_offset; - ArgsDumpTaskMetadata dump_metadata; -}; - -struct GraphBoundarySignature { - uint64_t buffer_size; - uint32_t shapes[MAX_TENSOR_DIMS]; - uint32_t strides[MAX_TENSOR_DIMS]; - uint16_t alias_rep; - uint8_t ndims; - uint8_t dtype; - uint8_t tag; - uint8_t manual_dep; - uint8_t is_contiguous; - uint8_t reserved; -}; - -struct GraphDefinition { - uint64_t full_key; - uint64_t content_hash; - uint64_t required_heap; - uint32_t total_bytes; - uint32_t task_count; - uint32_t edge_count; - uint32_t root_count; - uint32_t boundary_count; - uint32_t boundary_scalar_count; - uint32_t tensor_arg_count; - uint32_t scalar_arg_count; - uint32_t off_fanout_offsets; - uint32_t off_fanout_indices; - uint32_t off_fanin_offsets; - uint32_t off_fanin_indices; - uint32_t off_root_indices; - uint32_t off_node_offsets; - uint32_t off_nodes; - uint32_t off_tensors; - uint32_t off_tensor_sources; - uint32_t off_scalars; - uint32_t off_scalar_sources; - uint32_t off_boundary_signatures; -}; - -struct GraphSubmission { - uint64_t graph_key; - uint64_t execution_storage; - uint64_t execution_storage_bytes; - uint64_t local_execution; - uint32_t activation_gate; - uint32_t total_bytes; - uint32_t definition_offset; - uint32_t tensors_offset; - uint32_t tensor_count; - uint32_t scalars_offset; - uint32_t scalar_count; -}; - -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); - -inline GraphTensor graph_tensor_pack(const ChipTensor &tensor) { - GraphTensor packed{}; - packed.buffer_addr = tensor.buffer.addr; - packed.buffer_size = tensor.buffer.size; - packed.owner_task_id = tensor.owner_task_id.raw; - packed.start_offset = tensor.start_offset; - packed.extent_elem = tensor.extent_elem_cache; - packed.version = tensor.version; - for (uint32_t i = 0; i < tensor.ndims; ++i) { - packed.shapes[i] = tensor.shapes[i]; - packed.strides[i] = tensor.strides[i]; - } - packed.ndims = static_cast(tensor.ndims); - packed.dtype = static_cast(tensor.dtype); - packed.manual_dep = tensor.manual_dep ? 1 : 0; - packed.is_contiguous = tensor.is_contiguous ? 1 : 0; - packed.address_space = static_cast(tensor.address_space); - return packed; -} - -inline void graph_tensor_unpack(const GraphTensor &packed, ChipTensor *tensor) { - tensor->buffer = PTOBufferHandle{packed.buffer_addr, packed.buffer_size}; - tensor->owner_task_id = PTO2TaskId{packed.owner_task_id}; - tensor->start_offset = packed.start_offset; - tensor->extent_elem_cache = packed.extent_elem; - tensor->version = packed.version; - tensor->ndims = packed.ndims; - tensor->dtype = static_cast(packed.dtype); - tensor->manual_dep = packed.manual_dep != 0; - tensor->is_contiguous = packed.is_contiguous != 0; - tensor->address_space = static_cast(packed.address_space); - for (uint32_t i = 0; i < MAX_TENSOR_DIMS; ++i) { - tensor->shapes[i] = packed.shapes[i]; - tensor->strides[i] = packed.strides[i]; - } - for (uint8_t &byte : tensor->_pad_cl2) - byte = 0; -} - -inline bool graph_tensor_wire_valid(const GraphTensor &tensor) { - if (tensor.buffer_addr == 0 || tensor.ndims == 0 || tensor.ndims > MAX_TENSOR_DIMS || - tensor.dtype >= static_cast(DataType::DATA_TYPE_NUM) || tensor.manual_dep > 1 || - tensor.is_contiguous > 1 || tensor.address_space > 1) { - return false; - } - - uint64_t extent = 1; - uint64_t expected_stride = 1; - bool contiguous = true; - for (int32_t i = static_cast(tensor.ndims) - 1; i >= 0; --i) { - const uint64_t shape = tensor.shapes[i]; - const uint64_t stride = tensor.strides[i]; - if (shape == 0 || stride == 0) return false; - contiguous &= stride == expected_stride; - if (shape - 1 > (UINT64_MAX - extent) / stride || expected_stride > UINT64_MAX / shape) return false; - extent += (shape - 1) * stride; - expected_stride *= shape; - } - if (extent != tensor.extent_elem || contiguous != (tensor.is_contiguous != 0)) return false; - - const uint64_t element_size = get_element_size(static_cast(tensor.dtype)); - const uint64_t buffer_elements = tensor.buffer_size / element_size; - return tensor.start_offset <= buffer_elements && tensor.extent_elem <= buffer_elements - tensor.start_offset; -} - -template -inline const T *graph_definition_array(const GraphDefinition &definition, uint32_t offset, uint32_t count) { - if (offset == 0 || offset > definition.total_bytes || offset % alignof(T) != 0) return nullptr; - const size_t remaining = static_cast(definition.total_bytes - offset); - if (count > remaining / sizeof(T)) return nullptr; - return reinterpret_cast(reinterpret_cast(&definition) + offset); -} - -template -inline const T *graph_definition_ptr(const GraphDefinition &definition, uint32_t offset) { - return graph_definition_array(definition, offset, 1); -} - -inline GraphSubmission *graph_submission_from_slot(PTO2TaskSlotState &slot) { - return slot.task_kind == TaskKind::GRAPH ? static_cast(slot.graph_context) : nullptr; -} - -inline const GraphDefinition *graph_submission_definition(const GraphSubmission &submission) { - if (submission.definition_offset == 0 || submission.definition_offset % alignof(GraphDefinition) != 0 || - submission.definition_offset > submission.total_bytes || - sizeof(GraphDefinition) > submission.total_bytes - submission.definition_offset) { - return nullptr; - } - const auto *definition = reinterpret_cast( - reinterpret_cast(&submission) + submission.definition_offset - ); - if (definition->total_bytes < sizeof(GraphDefinition) || - definition->total_bytes > submission.total_bytes - submission.definition_offset) { - return nullptr; - } - return definition; -} - -inline bool graph_submission_wire_size_valid(const GraphSubmission &submission, size_t available_bytes) { - return available_bytes >= sizeof(GraphSubmission) && submission.total_bytes == available_bytes; -} - -inline const GraphTensor *graph_submission_tensors(const GraphSubmission &submission) { - if (submission.tensors_offset == 0 || submission.tensors_offset % alignof(GraphTensor) != 0 || - submission.tensors_offset > submission.total_bytes || - submission.tensor_count > (submission.total_bytes - submission.tensors_offset) / sizeof(GraphTensor)) { - return nullptr; - } - return reinterpret_cast( - reinterpret_cast(&submission) + submission.tensors_offset - ); -} - -inline const uint64_t *graph_submission_scalars(const GraphSubmission &submission) { - if (submission.scalar_count == 0) return nullptr; - if (submission.scalars_offset == 0 || submission.scalars_offset % alignof(uint64_t) != 0 || - submission.scalars_offset > submission.total_bytes || - submission.scalar_count > (submission.total_bytes - submission.scalars_offset) / sizeof(uint64_t)) { - return nullptr; - } - return reinterpret_cast( - reinterpret_cast(&submission) + submission.scalars_offset - ); -} - -enum class GraphExecutionState : uint8_t { - SUBMITTED = 0, - MATERIALIZING = 1, - PREPARED = 2, - ACTIVE = 3, - COMPLETED = 4, -}; - -enum class GraphMaterializeResult : uint8_t { - INVALID = 0, - BUSY = 1, - PENDING = 2, - PREPARED = 3, -}; - -enum class GraphTensorAddressSource : uint8_t { - BOUNDARY = 0, - INTERNAL = 1, -}; - -// Precomputed on the first materialization and retained next to the node -// storage. Affine replay walks this compact POD instead of re-reading and -// classifying GraphTensorSourceRef entries from the Definition. -struct GraphTensorAddressPatch { - uint64_t address_offset; - uint16_t source_index; - uint8_t source; - uint8_t reserved[5]; -}; - -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(sizeof(GraphTensorAddressPatch) == 16); - -struct GraphScalarPatch { - uint16_t node_index; - uint8_t node_scalar_index; - uint8_t boundary_scalar_index; -}; - -static_assert(std::is_trivially_copyable_v); -static_assert(std::is_standard_layout_v); -static_assert(sizeof(GraphScalarPatch) == 4); -static_assert(GRAPH_MAX_NODES <= UINT16_MAX); -static_assert(MAX_SCALAR_ARGS <= UINT8_MAX); - -struct alignas(64) GraphNodeStorage { - PTO2TaskDescriptor task; - PTO2TaskPayload payload; - PTO2TaskSlotState slot; -}; - -inline constexpr uint64_t GRAPH_EXECUTION_STORAGE_MAGIC = 0x4752415048455845ULL; -inline constexpr uint64_t GRAPH_EXECUTION_INITIALIZING = 1; - -struct GraphExecution { - uint64_t storage_magic{0}; - std::atomic state{GraphExecutionState::SUBMITTED}; - std::atomic materialize_busy{0}; - std::atomic remaining_nodes{0}; - std::atomic retired_nodes{0}; - // Incremental activation: nodes in [0, published_nodes) are fully - // materialized and registered, so a route pass may consider them. route_cursor - // is the next such node index a route pass will claim; roots below it have - // been pushed to the ready queue exactly once. Both advance monotonically and - // reset per (re)submission. - std::atomic published_nodes{0}; - std::atomic route_cursor{0}; - int32_t node_count{0}; - int32_t node_capacity{0}; - int32_t materialized_nodes{0}; - int32_t materialized_node_count{0}; - int32_t constructed_nodes{0}; - uint32_t tensor_patch_capacity{0}; - uint32_t materialized_tensor_patches{0}; - uint32_t materialized_tensor_patch_count{0}; - uint32_t scalar_patch_capacity{0}; - uint32_t materialized_scalar_patches{0}; - uint32_t materialized_scalar_patch_count{0}; - size_t allocation_bytes{0}; - size_t definition_capacity{0}; - uint64_t graph_key{0}; - uint64_t definition_hash{0}; - uint64_t materialized_graph_key{0}; - uint64_t materialized_definition_hash{0}; - uintptr_t materialized_outer_base{0}; - bool definition_affine_reuse{false}; - PTO2TaskSlotState *outer_slot{nullptr}; - GraphNodeStorage *nodes{nullptr}; - GraphNodeStorage *node_storage{nullptr}; - GraphTensorAddressPatch *tensor_patches{nullptr}; - GraphScalarPatch *scalar_patches{nullptr}; - void *definition_storage{nullptr}; - const GraphDefinition *definition{nullptr}; - const uint32_t *fanin_offsets{nullptr}; - const uint16_t *fanin_indices{nullptr}; - const GraphTensor *boundary_tensors{nullptr}; - uint32_t boundary_tensor_count{0}; - const uint64_t *boundary_scalars{nullptr}; - uint32_t boundary_scalar_count{0}; -}; - -static_assert(offsetof(GraphExecution, storage_magic) == 0); -static_assert(sizeof(GraphExecution::storage_magic) == sizeof(uint64_t)); -static_assert(std::is_trivially_destructible_v); -static_assert(std::is_trivially_destructible_v); - -inline bool graph_execution_storage_layout( - int32_t node_capacity, uint32_t tensor_patch_capacity, uint32_t scalar_patch_capacity, size_t definition_capacity, - size_t *nodes_offset, size_t *tensor_patches_offset, size_t *scalar_patches_offset, size_t *definition_offset, - size_t *storage_bytes -) { - if (nodes_offset == nullptr || tensor_patches_offset == nullptr || scalar_patches_offset == nullptr || - definition_offset == nullptr || storage_bytes == nullptr || node_capacity <= 0 || - static_cast(node_capacity) > SIZE_MAX / sizeof(GraphNodeStorage) || - tensor_patch_capacity > GRAPH_MAX_NODES * MAX_TENSOR_ARGS || - scalar_patch_capacity > GRAPH_MAX_NODES * MAX_SCALAR_ARGS) { - return false; - } - auto checked_align_up = [](size_t value, size_t alignment, size_t *result) { - if (alignment == 0 || value > SIZE_MAX - (alignment - 1)) return false; - *result = (value + alignment - 1) & ~(alignment - 1); - return true; - }; - const size_t nodes_bytes = static_cast(node_capacity) * sizeof(GraphNodeStorage); - const size_t tensor_patches_bytes = static_cast(tensor_patch_capacity) * sizeof(GraphTensorAddressPatch); - const size_t scalar_patches_bytes = static_cast(scalar_patch_capacity) * sizeof(GraphScalarPatch); - if (!checked_align_up(sizeof(GraphExecution), alignof(GraphNodeStorage), nodes_offset) || - *nodes_offset > SIZE_MAX - nodes_bytes || - !checked_align_up(*nodes_offset + nodes_bytes, alignof(GraphTensorAddressPatch), tensor_patches_offset) || - *tensor_patches_offset > SIZE_MAX - tensor_patches_bytes || - !checked_align_up( - *tensor_patches_offset + tensor_patches_bytes, alignof(GraphScalarPatch), scalar_patches_offset - ) || - *scalar_patches_offset > SIZE_MAX - scalar_patches_bytes || - !checked_align_up(*scalar_patches_offset + scalar_patches_bytes, alignof(GraphDefinition), definition_offset) || - *definition_offset > SIZE_MAX - definition_capacity) { - return false; - } - return checked_align_up(*definition_offset + definition_capacity, alignof(GraphNodeStorage), storage_bytes); -} - -inline bool graph_execution_storage_bytes( - int32_t node_capacity, uint32_t tensor_patch_capacity, uint32_t scalar_patch_capacity, size_t definition_capacity, - size_t *storage_bytes -) { - size_t nodes_offset = 0; - size_t tensor_patches_offset = 0; - size_t scalar_patches_offset = 0; - size_t definition_offset = 0; - return graph_execution_storage_layout( - node_capacity, tensor_patch_capacity, scalar_patch_capacity, definition_capacity, &nodes_offset, - &tensor_patches_offset, &scalar_patches_offset, &definition_offset, storage_bytes - ); -} - -GraphExecution *graph_execution_localize(PTO2TaskSlotState &outer_slot); -GraphMaterializeResult graph_execution_materialize_slice( - PTO2TaskSlotState &outer_slot, GraphExecution &execution, int32_t max_nodes, int32_t *nodes_materialized = nullptr -); - -inline GraphExecution *graph_execution_from_slot(PTO2TaskSlotState &slot) { - return slot.task_kind == TaskKind::GRAPH_NODE ? static_cast(slot.graph_context) : nullptr; -} - -inline bool graph_execution_complete_node(GraphExecution &execution) { - return execution.remaining_nodes.fetch_sub(1, std::memory_order_acq_rel) == 1; -} - -inline void graph_execution_mark_completed(GraphExecution &execution) { - execution.state.store(GraphExecutionState::COMPLETED, std::memory_order_release); -} - -inline void graph_execution_retire_node(GraphExecution &execution) { - execution.retired_nodes.fetch_add(1, std::memory_order_release); -} - -inline bool graph_submission_signal(GraphSubmission &submission, uint32_t bit) { - constexpr uint32_t BOTH = 0x3; - uint32_t observed = __atomic_fetch_or(&submission.activation_gate, bit, __ATOMIC_ACQ_REL); - return observed != BOTH && (observed | bit) == BOTH; -} - -inline GraphExecution *graph_submission_local_execution(GraphSubmission &submission) { - uint64_t raw = __atomic_load_n(&submission.local_execution, __ATOMIC_ACQUIRE); - if (raw <= GRAPH_EXECUTION_INITIALIZING) return nullptr; - return reinterpret_cast(static_cast(raw)); -} - -inline bool graph_submission_execution_initializing(const GraphSubmission &submission) { - return __atomic_load_n(&submission.local_execution, __ATOMIC_ACQUIRE) == GRAPH_EXECUTION_INITIALIZING; -} +#include "host_build_graph/graph_execution.h" diff --git a/src/a5/runtime/host_build_graph/runtime/graph_host_state.h b/src/a5/runtime/host_build_graph/runtime/graph_host_state.h index 4d869b1c4d..885c97660e 100644 --- a/src/a5/runtime/host_build_graph/runtime/graph_host_state.h +++ b/src/a5/runtime/host_build_graph/runtime/graph_host_state.h @@ -11,27 +11,4 @@ #pragma once -#include -#include -#include - -struct PTO2TaskSlotState; -struct GraphHostState; - -inline constexpr size_t GRAPH_MAX_DEFINITIONS = 16; - -struct GraphHostStateDeleter { - void operator()(GraphHostState *state) const noexcept; -}; - -using GraphHostStatePtr = std::unique_ptr; - -struct GraphHostUpload { - PTO2TaskSlotState *outer_slot; - std::byte *data; - size_t bytes; -}; - -GraphHostStatePtr make_graph_host_state(); -size_t graph_host_upload_count(const GraphHostState &state); -std::optional graph_host_upload(GraphHostState &state, size_t index); +#include "host_build_graph/graph_host_state.h" diff --git a/src/a5/runtime/host_build_graph/runtime/scheduler/graph_execution.cpp b/src/a5/runtime/host_build_graph/runtime/scheduler/graph_execution.cpp deleted file mode 100644 index c29fbae0fd..0000000000 --- a/src/a5/runtime/host_build_graph/runtime/scheduler/graph_execution.cpp +++ /dev/null @@ -1,613 +0,0 @@ -/* - * Copyright (c) PyPTO Contributors. - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of - * CANN Open Software License Agreement Version 2.0 (the "License"). - * Please refer to the License for details. You may not use this file except in compliance with the License. - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. - * See LICENSE in the root of the software repository for the full text of the License. - * ----------------------------------------------------------------------------------------------------------- - */ - -#include "graph_execution.h" - -#include -#include -#include - -#include "graph_cache.h" - -namespace { - -void destroy_execution_nodes(GraphExecution *execution) { - for (int32_t i = 0; i < execution->constructed_nodes; ++i) { - execution->node_storage[i].~GraphNodeStorage(); - } - execution->constructed_nodes = 0; -} - -void reset_execution( - GraphExecution *execution, int32_t node_count, uint32_t tensor_patch_count, uint32_t scalar_patch_count, - uint64_t graph_key, uint64_t definition_hash -) { - size_t nodes_offset = 0; - size_t tensor_patches_offset = 0; - size_t scalar_patches_offset = 0; - size_t definition_offset = 0; - size_t bytes = 0; - if (!graph_execution_storage_layout( - execution->node_capacity, execution->tensor_patch_capacity, execution->scalar_patch_capacity, - execution->definition_capacity, &nodes_offset, &tensor_patches_offset, &scalar_patches_offset, - &definition_offset, &bytes - )) { - return; - } - execution->definition_affine_reuse = graph_key != 0 && execution->materialized_graph_key == graph_key && - execution->materialized_definition_hash == definition_hash && - execution->materialized_node_count == node_count && - execution->materialized_tensor_patch_count == tensor_patch_count && - execution->materialized_scalar_patch_count <= scalar_patch_count && - execution->constructed_nodes >= node_count; - execution->state.store(GraphExecutionState::SUBMITTED, std::memory_order_relaxed); - execution->materialize_busy.store(0, std::memory_order_relaxed); - execution->remaining_nodes.store(node_count, std::memory_order_relaxed); - execution->retired_nodes.store(0, std::memory_order_relaxed); - execution->published_nodes.store(0, std::memory_order_relaxed); - execution->route_cursor.store(0, std::memory_order_relaxed); - execution->node_count = node_count; - execution->materialized_nodes = 0; - execution->materialized_tensor_patches = 0; - execution->materialized_scalar_patches = 0; - execution->allocation_bytes = bytes; - execution->graph_key = graph_key; - execution->definition_hash = definition_hash; - execution->outer_slot = nullptr; - execution->nodes = nullptr; - execution->node_storage = - reinterpret_cast(reinterpret_cast(execution) + nodes_offset); - execution->tensor_patches = - reinterpret_cast(reinterpret_cast(execution) + tensor_patches_offset); - execution->scalar_patches = - reinterpret_cast(reinterpret_cast(execution) + scalar_patches_offset); - execution->definition_storage = reinterpret_cast(execution) + definition_offset; - if (!execution->definition_affine_reuse) { - execution->definition = nullptr; - execution->fanin_offsets = nullptr; - execution->fanin_indices = nullptr; - } - execution->boundary_tensors = nullptr; - execution->boundary_tensor_count = 0; - execution->boundary_scalars = nullptr; - execution->boundary_scalar_count = 0; -} - -bool reusable_execution_header_valid(const GraphExecution &execution, size_t storage_bytes) { - if (execution.storage_magic != GRAPH_EXECUTION_STORAGE_MAGIC || execution.node_capacity <= 0 || - execution.node_capacity > static_cast(GRAPH_MAX_NODES) || execution.definition_capacity == 0 || - execution.tensor_patch_capacity > GRAPH_MAX_NODES * MAX_TENSOR_ARGS || - execution.scalar_patch_capacity > GRAPH_MAX_NODES * MAX_SCALAR_ARGS || - execution.materialized_tensor_patch_count > execution.tensor_patch_capacity || - execution.materialized_scalar_patch_count > execution.scalar_patch_capacity || - execution.constructed_nodes < 0 || execution.constructed_nodes > execution.node_capacity || - execution.node_count <= 0 || execution.node_count > execution.node_capacity || - execution.state.load(std::memory_order_acquire) != GraphExecutionState::COMPLETED || - execution.retired_nodes.load(std::memory_order_acquire) < execution.node_count) { - return false; - } - size_t expected_bytes = 0; - return graph_execution_storage_bytes( - execution.node_capacity, execution.tensor_patch_capacity, execution.scalar_patch_capacity, - execution.definition_capacity, &expected_bytes - ) && - execution.allocation_bytes == expected_bytes && expected_bytes <= storage_bytes; -} - -GraphExecution *acquire_host_execution_storage( - GraphSubmission &submission, int32_t node_count, uint64_t graph_key, uint64_t definition_hash, - uint32_t tensor_patch_count, uint32_t scalar_patch_count, size_t definition_bytes -) { - if (node_count <= 0 || node_count > static_cast(GRAPH_MAX_NODES) || definition_bytes == 0 || - submission.execution_storage == 0 || submission.execution_storage_bytes > SIZE_MAX || - submission.execution_storage % alignof(GraphNodeStorage) != 0) { - return nullptr; - } - const size_t storage_bytes = static_cast(submission.execution_storage_bytes); - size_t required_bytes = 0; - if (!graph_execution_storage_bytes( - node_count, tensor_patch_count, scalar_patch_count, definition_bytes, &required_bytes - ) || - required_bytes > storage_bytes) { - return nullptr; - } - - auto *execution = reinterpret_cast(static_cast(submission.execution_storage)); - uint64_t observed_magic = 0; - std::memcpy(&observed_magic, execution, sizeof(observed_magic)); - const bool has_existing_execution = observed_magic == GRAPH_EXECUTION_STORAGE_MAGIC; - const bool valid_header = has_existing_execution && reusable_execution_header_valid(*execution, storage_bytes); - if (has_existing_execution && !valid_header) return nullptr; - const bool capacities_fit = valid_header && execution->node_capacity >= node_count && - execution->tensor_patch_capacity >= tensor_patch_count && - execution->scalar_patch_capacity >= scalar_patch_count && - execution->definition_capacity >= definition_bytes; - if (!capacities_fit) { - if (valid_header) destroy_execution_nodes(execution); - execution = new (execution) GraphExecution{}; - execution->node_capacity = node_count; - execution->tensor_patch_capacity = tensor_patch_count; - execution->scalar_patch_capacity = scalar_patch_count; - execution->definition_capacity = definition_bytes; - execution->allocation_bytes = required_bytes; - } - reset_execution(execution, node_count, tensor_patch_count, scalar_patch_count, graph_key, definition_hash); - execution->storage_magic = GRAPH_EXECUTION_STORAGE_MAGIC; - return execution; -} - -void reset_graph_payload(PTO2TaskPayload &payload) { - payload.fanin_count = 0; - payload.predicate = DispatchPredicate{}; - payload.early_dispatch_state.store(PTO2_EARLY_DISPATCH_NONE, std::memory_order_relaxed); - for (int w = 0; w < PTO2_EARLY_DISPATCH_CORE_MASK_WORDS; ++w) { - payload.staged_core_mask[w].store(0, std::memory_order_relaxed); - } - payload.dispatch_fanin.store(0, std::memory_order_relaxed); - payload.dispatch_propagated.store(0, std::memory_order_relaxed); - payload.published_block_count.store(0, std::memory_order_relaxed); - payload.early_dispatch_launch_state.store(PTO2_EARLY_DISPATCH_LAUNCH_NONE, std::memory_order_relaxed); - payload.running_slot_count.store(0, std::memory_order_relaxed); - payload.early_sync_drain_state.store(PTO2_EARLY_SYNC_DRAIN_NONE, std::memory_order_relaxed); -} - -bool bind_graph_topology(GraphExecution &execution) { - if (execution.definition == nullptr) return false; - const GraphDefinition &definition = *execution.definition; - if (definition.boundary_scalar_count > MAX_SCALAR_ARGS) return false; - const uint32_t *fanin_offsets = - graph_definition_array(definition, definition.off_fanin_offsets, definition.task_count + 1); - const uint16_t *fanin_indices = - definition.edge_count == 0 ? - nullptr : - graph_definition_array(definition, definition.off_fanin_indices, definition.edge_count); - const uint32_t *fanout_offsets = - graph_definition_array(definition, definition.off_fanout_offsets, definition.task_count + 1); - const uint16_t *fanout_indices = - definition.edge_count == 0 ? - nullptr : - graph_definition_array(definition, definition.off_fanout_indices, definition.edge_count); - const uint16_t *roots = - graph_definition_array(definition, definition.off_root_indices, definition.root_count); - const GraphNodeDefinition *nodes = - graph_definition_array(definition, definition.off_nodes, definition.task_count); - const uint64_t *node_offsets = - graph_definition_array(definition, definition.off_node_offsets, definition.task_count); - if (fanin_offsets == nullptr || fanout_offsets == nullptr || roots == nullptr || nodes == nullptr || - node_offsets == nullptr || - (definition.edge_count != 0 && (fanin_indices == nullptr || fanout_indices == nullptr)) || - fanin_offsets[0] != 0 || fanout_offsets[0] != 0 || - fanin_offsets[definition.task_count] != definition.edge_count || - fanout_offsets[definition.task_count] != definition.edge_count) { - return false; - } - - uint64_t required_heap = 0; - constexpr uint8_t VALID_ACTIVE_MASK = (1U << PTO2_SUBTASK_SLOT_COUNT) - 1U; - for (uint32_t i = 0; i < definition.task_count; ++i) { - const GraphNodeDefinition &node = nodes[i]; - if (node_offsets[i] != required_heap || node.total_output_size < 0 || node.tensor_count < 0 || - node.tensor_count > MAX_TENSOR_ARGS || node.scalar_count < 0 || node.scalar_count > MAX_SCALAR_ARGS || - node.tensor_offset > definition.tensor_arg_count || - static_cast(node.tensor_count) > definition.tensor_arg_count - node.tensor_offset || - node.scalar_offset > definition.scalar_arg_count || - static_cast(node.scalar_count) > definition.scalar_arg_count - node.scalar_offset || - (node.active_mask & ~VALID_ACTIVE_MASK) != 0 || node.logical_block_num <= 0 || - node.total_required_subtasks < 0) { - return false; - } - for (int32_t slot = 0; slot < PTO2_SUBTASK_SLOT_COUNT; ++slot) { - const bool active = (node.active_mask & (1U << slot)) != 0; - if (active != (node.kernel_id[slot] != INVALID_KERNEL_ID)) return false; - } - const uint64_t output_bytes = PTO2_ALIGN_UP(static_cast(node.total_output_size), PTO2_ALIGN_SIZE); - if (output_bytes > definition.required_heap - required_heap) return false; - required_heap += output_bytes; - } - if (required_heap != definition.required_heap) return false; - - uint32_t observed_roots = 0; - for (uint32_t consumer = 0; consumer < definition.task_count; ++consumer) { - const uint32_t begin = fanin_offsets[consumer]; - const uint32_t end = fanin_offsets[consumer + 1]; - if (begin > end || end > definition.edge_count) return false; - if (begin == end) observed_roots++; - for (uint32_t edge = begin; edge < end; ++edge) { - if (fanin_indices[edge] >= consumer) return false; - } - } - if (observed_roots != definition.root_count) return false; - for (uint32_t i = 0; i < definition.root_count; ++i) { - const uint16_t root = roots[i]; - if (root >= definition.task_count || fanin_offsets[root] != fanin_offsets[root + 1]) return false; - } - for (uint32_t producer = 0; producer < definition.task_count; ++producer) { - const uint32_t begin = fanout_offsets[producer]; - const uint32_t end = fanout_offsets[producer + 1]; - if (begin > end || end > definition.edge_count) return false; - for (uint32_t edge = begin; edge < end; ++edge) { - if (fanout_indices[edge] <= producer || fanout_indices[edge] >= definition.task_count) return false; - } - } - - execution.fanin_offsets = fanin_offsets; - execution.fanin_indices = fanin_indices; - return true; -} - -bool graph_definition_hash_matches(const GraphDefinition &definition) { - if (definition.content_hash == 0 || definition.total_bytes < sizeof(GraphDefinition)) return false; - constexpr size_t HASH_OFFSET = offsetof(GraphDefinition, content_hash); - constexpr size_t HASH_END = HASH_OFFSET + sizeof(GraphDefinition::content_hash); - const auto *bytes = reinterpret_cast(&definition); - uint64_t hash = graph_hash_bytes(1469598103934665603ULL, bytes, HASH_OFFSET); - const uint64_t zero_hash = 0; - hash = graph_hash_bytes(hash, &zero_hash, sizeof(zero_hash)); - hash = graph_hash_bytes(hash, bytes + HASH_END, definition.total_bytes - HASH_END); - return hash == definition.content_hash; -} - -} // namespace - -GraphExecution *graph_execution_localize(PTO2TaskSlotState &outer_slot) { - GraphSubmission *submission = graph_submission_from_slot(outer_slot); - if (submission == nullptr) return nullptr; - if (GraphExecution *existing = graph_submission_local_execution(*submission)) return existing; - if (graph_submission_execution_initializing(*submission)) return nullptr; - - const GraphDefinition *definition = graph_submission_definition(*submission); - const GraphTensor *boundary_tensors = graph_submission_tensors(*submission); - const uint64_t *boundary_scalars = graph_submission_scalars(*submission); - const size_t boundary_tensor_end = static_cast(submission->tensors_offset) + - static_cast(submission->tensor_count) * sizeof(GraphTensor); - if (definition == nullptr || definition->total_bytes == 0 || definition->task_count == 0 || - definition->task_count > GRAPH_MAX_NODES || - definition->total_bytes > submission->total_bytes - submission->definition_offset || - submission->graph_key != definition->full_key || submission->tensor_count != definition->boundary_count || - submission->scalar_count != definition->boundary_scalar_count || boundary_tensors == nullptr || - (submission->scalar_count != 0 && boundary_scalars == nullptr) || - (submission->scalar_count != 0 && submission->scalars_offset < boundary_tensor_end) || - submission->tensors_offset < submission->definition_offset + definition->total_bytes || - !graph_definition_hash_matches(*definition) || outer_slot.task == nullptr || - outer_slot.task->packed_buffer_base == nullptr || outer_slot.task->packed_buffer_end == nullptr) { - return nullptr; - } - const uintptr_t outer_base = reinterpret_cast(outer_slot.task->packed_buffer_base); - const uintptr_t outer_end = reinterpret_cast(outer_slot.task->packed_buffer_end); - if (outer_end < outer_base || definition->required_heap > outer_end - outer_base) return nullptr; - for (uint32_t i = 0; i < submission->tensor_count; ++i) { - if (!graph_tensor_wire_valid(boundary_tensors[i])) return nullptr; - } - - uint64_t expected = 0; - if (!__atomic_compare_exchange_n( - &submission->local_execution, &expected, GRAPH_EXECUTION_INITIALIZING, false, __ATOMIC_ACQ_REL, - __ATOMIC_ACQUIRE - )) { - return expected > GRAPH_EXECUTION_INITIALIZING ? - reinterpret_cast(static_cast(expected)) : - nullptr; - } - - GraphExecution *execution = acquire_host_execution_storage( - *submission, static_cast(definition->task_count), submission->graph_key, definition->content_hash, - definition->tensor_arg_count, definition->scalar_arg_count, definition->total_bytes - ); - if (execution == nullptr) { - __atomic_store_n(&submission->local_execution, 0, __ATOMIC_RELEASE); - return nullptr; - } - - if (!execution->definition_affine_reuse) { - std::memcpy(execution->definition_storage, definition, definition->total_bytes); - execution->definition = static_cast(execution->definition_storage); - if (!bind_graph_topology(*execution)) { - execution->retired_nodes.store(execution->node_count, std::memory_order_relaxed); - execution->state.store(GraphExecutionState::COMPLETED, std::memory_order_release); - __atomic_store_n(&submission->local_execution, 0, __ATOMIC_RELEASE); - return nullptr; - } - } - execution->boundary_tensors = boundary_tensors; - execution->boundary_tensor_count = submission->tensor_count; - execution->boundary_scalars = boundary_scalars; - execution->boundary_scalar_count = submission->scalar_count; - execution->outer_slot = &outer_slot; - - const uint64_t desired = static_cast(reinterpret_cast(execution)); - __atomic_store_n(&submission->local_execution, desired, __ATOMIC_RELEASE); - return execution; -} - -GraphMaterializeResult graph_execution_materialize_slice( - PTO2TaskSlotState &outer_slot, GraphExecution &execution, int32_t max_nodes, int32_t *nodes_materialized -) { - if (nodes_materialized != nullptr) *nodes_materialized = 0; - if (outer_slot.task_kind != TaskKind::GRAPH || outer_slot.task == nullptr || - outer_slot.task->packed_buffer_base == nullptr || max_nodes <= 0 || execution.definition == nullptr || - execution.node_storage == nullptr || execution.tensor_patches == nullptr || - execution.scalar_patches == nullptr) { - return GraphMaterializeResult::INVALID; - } - - GraphExecutionState state = execution.state.load(std::memory_order_acquire); - if (state >= GraphExecutionState::PREPARED) return GraphMaterializeResult::PREPARED; - - uint8_t expected_busy = 0; - if (!execution.materialize_busy.compare_exchange_strong( - expected_busy, 1, std::memory_order_acq_rel, std::memory_order_acquire - )) { - return GraphMaterializeResult::BUSY; - } - - state = execution.state.load(std::memory_order_acquire); - if (state == GraphExecutionState::SUBMITTED) { - GraphExecutionState expected = GraphExecutionState::SUBMITTED; - if (!execution.state.compare_exchange_strong( - expected, GraphExecutionState::MATERIALIZING, std::memory_order_acq_rel, std::memory_order_acquire - )) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::BUSY; - } - // Incremental activation reads producer slots through execution.nodes - // while the graph is still materializing, so publish the storage base - // once, before the first range. Topological node order guarantees every - // producer index a materialized node references is already constructed, - // and materialize_busy serializes this with any concurrent slice. - execution.nodes = execution.node_storage; - } else if (state != GraphExecutionState::MATERIALIZING) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - - const bool affine_reuse = execution.definition_affine_reuse; - const GraphDefinition &definition = *execution.definition; - const GraphNodeDefinition *nodes = nullptr; - const uint64_t *node_offsets = nullptr; - const GraphTensor *definition_tensors = nullptr; - const GraphTensorSourceRef *tensor_sources = nullptr; - const uint64_t *definition_scalars = nullptr; - const GraphScalarSourceRef *scalar_sources = nullptr; - if (!affine_reuse) { - nodes = graph_definition_array(definition, definition.off_nodes, definition.task_count); - node_offsets = graph_definition_array(definition, definition.off_node_offsets, definition.task_count); - definition_tensors = - definition.tensor_arg_count == 0 ? - nullptr : - graph_definition_array(definition, definition.off_tensors, definition.tensor_arg_count); - tensor_sources = definition.tensor_arg_count == 0 ? - nullptr : - graph_definition_array( - definition, definition.off_tensor_sources, definition.tensor_arg_count - ); - definition_scalars = - definition.scalar_arg_count == 0 ? - nullptr : - graph_definition_array(definition, definition.off_scalars, definition.scalar_arg_count); - scalar_sources = definition.scalar_arg_count == 0 ? - nullptr : - graph_definition_array( - definition, definition.off_scalar_sources, definition.scalar_arg_count - ); - if (nodes == nullptr || node_offsets == nullptr || - (definition.tensor_arg_count != 0 && (definition_tensors == nullptr || tensor_sources == nullptr)) || - (definition.scalar_arg_count != 0 && (definition_scalars == nullptr || scalar_sources == nullptr))) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - } - - const int32_t first = execution.materialized_nodes; - const int32_t last = std::min(execution.node_count, first + max_nodes); - const uintptr_t outer_base = reinterpret_cast(outer_slot.task->packed_buffer_base); - for (int32_t i = first; i < last; ++i) { - GraphNodeStorage *storage = &execution.node_storage[i]; - if (i >= execution.constructed_nodes) { - storage = new (storage) GraphNodeStorage; - execution.constructed_nodes++; - } - PTO2TaskDescriptor &task = storage->task; - PTO2TaskPayload &payload = storage->payload; - PTO2TaskSlotState &slot = storage->slot; - - const uint32_t synthetic_local = - (static_cast(outer_slot.task->task_id.local()) << 10) | static_cast(i); - task.task_id = PTO2TaskId::make(1, synthetic_local); - uint64_t node_offset = 0; - uint64_t output_bytes = 0; - if (affine_reuse) { - const uintptr_t previous_base = reinterpret_cast(task.packed_buffer_base); - const uintptr_t previous_end = reinterpret_cast(task.packed_buffer_end); - node_offset = previous_base - execution.materialized_outer_base; - output_bytes = previous_end - previous_base; - } else { - const GraphNodeDefinition &source = nodes[i]; - node_offset = node_offsets[i]; - output_bytes = PTO2_ALIGN_UP(static_cast(source.total_output_size), PTO2_ALIGN_SIZE); - for (int k = 0; k < PTO2_SUBTASK_SLOT_COUNT; ++k) - task.kernel_id[k] = source.kernel_id[k]; - } - task.packed_buffer_base = reinterpret_cast(outer_base + node_offset); - task.packed_buffer_end = reinterpret_cast(outer_base + node_offset + output_bytes); - - slot.reset_for_reuse(affine_reuse); - slot.task_state.store(PTO2_TASK_PENDING, std::memory_order_relaxed); - if (!affine_reuse) { - const GraphNodeDefinition &source = nodes[i]; - slot.bind_buffers(&payload, &task); - slot.active_mask = ActiveMask(source.active_mask); - slot.task_attrs = TaskAttrs(source.task_attrs); - slot.total_required_subtasks = source.total_required_subtasks; - slot.logical_block_num = source.logical_block_num; - slot.graph_node_index = i; - slot.task_kind = TaskKind::GRAPH_NODE; - slot.graph_context = &execution; - payload.tensor_count = source.tensor_count; - payload.scalar_count = source.scalar_count; - payload.dump_metadata = source.dump_metadata; - if (source.tensor_count < 0 || source.tensor_count > MAX_TENSOR_ARGS || source.scalar_count < 0 || - source.scalar_count > MAX_SCALAR_ARGS || - static_cast(source.tensor_count) > definition.tensor_arg_count || - static_cast(source.scalar_count) > definition.scalar_arg_count || - source.tensor_offset > definition.tensor_arg_count - static_cast(source.tensor_count) || - source.scalar_offset > definition.scalar_arg_count - static_cast(source.scalar_count)) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - for (int32_t j = 0; j < source.tensor_count; ++j) { - const uint32_t tensor_index = source.tensor_offset + static_cast(j); - ChipTensor &tensor = payload.tensors[j]; - GraphTensor rebound = definition_tensors[tensor_index]; - GraphTensorAddressPatch patch{}; - if (!graph_tensor_wire_valid(rebound)) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - const GraphTensorSourceRef &ref = tensor_sources[tensor_index]; - if (ref.source == static_cast(GraphTensorSource::BOUNDARY_EXACT)) { - if (ref.source_index >= execution.boundary_tensor_count || ref.packed_offset != 0) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - rebound = execution.boundary_tensors[ref.source_index]; - patch.source = static_cast(GraphTensorAddressSource::BOUNDARY); - patch.source_index = ref.source_index; - } else if (ref.source == static_cast(GraphTensorSource::BOUNDARY_VIEW)) { - if (ref.source_index >= execution.boundary_tensor_count) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - const GraphTensor &boundary = execution.boundary_tensors[ref.source_index]; - if (ref.packed_offset > UINT64_MAX - boundary.start_offset) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - rebound.buffer_addr = boundary.buffer_addr; - rebound.buffer_size = boundary.buffer_size; - rebound.owner_task_id = boundary.owner_task_id; - rebound.start_offset = boundary.start_offset + ref.packed_offset; - rebound.version = boundary.version; - rebound.address_space = boundary.address_space; - patch.source = static_cast(GraphTensorAddressSource::BOUNDARY); - patch.source_index = ref.source_index; - } else if (ref.source == static_cast(GraphTensorSource::INTERNAL) || - ref.source == static_cast(GraphTensorSource::OWN_OUTPUT)) { - const bool own_output = ref.source == static_cast(GraphTensorSource::OWN_OUTPUT); - const int32_t producer_index = own_output ? i : static_cast(ref.source_index); - if (producer_index < 0 || producer_index > i || (own_output && ref.source_index != i) || - (!own_output && producer_index == i)) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - PTO2TaskDescriptor &producer = execution.node_storage[producer_index].task; - const uint64_t producer_bytes = static_cast(nodes[producer_index].total_output_size); - const uintptr_t producer_base = reinterpret_cast(producer.packed_buffer_base); - if (ref.packed_offset > producer_bytes || - rebound.buffer_size > producer_bytes - ref.packed_offset || - ref.packed_offset > UINTPTR_MAX - producer_base || - ref.packed_offset > UINT64_MAX - node_offsets[producer_index]) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - rebound.buffer_addr = producer_base + ref.packed_offset; - rebound.owner_task_id = producer.task_id.raw; - patch.source = static_cast(GraphTensorAddressSource::INTERNAL); - patch.address_offset = node_offsets[producer_index] + ref.packed_offset; - } else { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - if (!graph_tensor_wire_valid(rebound) || - execution.materialized_tensor_patches >= execution.tensor_patch_capacity) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - execution.tensor_patches[execution.materialized_tensor_patches++] = patch; - graph_tensor_unpack(rebound, &tensor); - } - for (int32_t j = 0; j < source.scalar_count; ++j) { - const uint32_t scalar_index = source.scalar_offset + static_cast(j); - const GraphScalarSourceRef &ref = scalar_sources[scalar_index]; - if (ref.source == static_cast(GraphScalarSource::STATIC_VALUE)) { - payload.scalars[j] = definition_scalars[scalar_index]; - } else if (ref.source == static_cast(GraphScalarSource::BOUNDARY)) { - if (ref.source_index >= execution.boundary_scalar_count || execution.boundary_scalars == nullptr || - execution.materialized_scalar_patches >= execution.scalar_patch_capacity) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - payload.scalars[j] = execution.boundary_scalars[ref.source_index]; - execution.scalar_patches[execution.materialized_scalar_patches++] = GraphScalarPatch{ - static_cast(i), static_cast(j), static_cast(ref.source_index) - }; - } else { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - } - } else { - for (int32_t j = 0; j < payload.tensor_count; ++j) { - if (execution.materialized_tensor_patches >= execution.materialized_tensor_patch_count) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - const GraphTensorAddressPatch &patch = - execution.tensor_patches[execution.materialized_tensor_patches++]; - if (patch.source == static_cast(GraphTensorAddressSource::BOUNDARY)) { - payload.tensors[j].buffer.addr = execution.boundary_tensors[patch.source_index].buffer_addr; - } else { - payload.tensors[j].buffer.addr = outer_base + patch.address_offset; - } - } - while (execution.materialized_scalar_patches < execution.materialized_scalar_patch_count) { - const GraphScalarPatch &patch = execution.scalar_patches[execution.materialized_scalar_patches]; - if (patch.node_index != static_cast(i)) break; - if (patch.node_scalar_index >= payload.scalar_count || - patch.boundary_scalar_index >= execution.boundary_scalar_count || - execution.boundary_scalars == nullptr) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - payload.scalars[patch.node_scalar_index] = execution.boundary_scalars[patch.boundary_scalar_index]; - execution.materialized_scalar_patches++; - } - } - reset_graph_payload(payload); - } - execution.materialized_nodes = last; - if (nodes_materialized != nullptr) *nodes_materialized = last - first; - - if (last < execution.node_count) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::PENDING; - } - - const uint32_t expected_tensor_patches = - affine_reuse ? execution.materialized_tensor_patch_count : definition.tensor_arg_count; - if (execution.materialized_tensor_patches != expected_tensor_patches) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - if (affine_reuse && execution.materialized_scalar_patches != execution.materialized_scalar_patch_count) { - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::INVALID; - } - - execution.materialized_graph_key = execution.graph_key; - execution.materialized_definition_hash = execution.definition_hash; - execution.materialized_node_count = execution.node_count; - execution.materialized_tensor_patch_count = execution.materialized_tensor_patches; - execution.materialized_scalar_patch_count = execution.materialized_scalar_patches; - execution.materialized_outer_base = outer_base; - execution.state.store(GraphExecutionState::PREPARED, std::memory_order_release); - execution.materialize_busy.store(0, std::memory_order_release); - return GraphMaterializeResult::PREPARED; -} diff --git a/src/common/host_build_graph/docs/GRAPH_EXECUTION.md b/src/common/host_build_graph/docs/GRAPH_EXECUTION.md new file mode 100644 index 0000000000..771548716d --- /dev/null +++ b/src/common/host_build_graph/docs/GRAPH_EXECUTION.md @@ -0,0 +1,387 @@ +# Graph Execution + +Graph Execution is available only in the `host_build_graph` runtime. A Graph is +a composite incore task: it is submitted and completed once like an AIC, AIV, +MIX, or SPMD task, but contains a recorded task DAG. + +Every invocation places exactly one `GRAPH` task in the host task window. The +first invocation records the DAG off the ring — its internal submissions build +host-only node metadata and reserve scratch output buffers instead of consuming +task-window slots — then emits the outer `GRAPH` task from the freshly built +Definition. Later invocations reuse the cached Definition and emit the same one +`GRAPH` task directly. In both cases the device Scheduler expands the saved +topology and dispatches the internal nodes; the Host Orchestrator never submits +those nodes as ring tasks. + +A recording that hits an unsupported construct is discarded and the body re-runs +on the ordinary task-submit path so its work is still submitted; the internal +nodes then occupy the ring only for that one fallback invocation. + +## API + +A Graph uses `CoreTaskArgs`, the existing incore argument type: + +```cpp +void graph_function(const CoreTaskArgs &args, int variant) { + const ChipTensor &input = args.tensor(0).ref(); + const ChipTensor &weight = args.tensor(1).ref(); + const ChipTensor &output = args.tensor(2).ref(); + + const std::array shape{input.shapes[0]}; + TensorCreateInfo intermediate( + shape.data(), static_cast(shape.size()), input.dtype + ); + + CoreTaskArgs matmul_args; + matmul_args.add_input(input, weight); + matmul_args.add_output(intermediate); + matmul_args.copy_scalars_from(args, 0, 1); // current invocation's value + TaskOutputTensors matmul = rt_submit_aic_task( + variant == 0 ? FUNC_MATMUL : FUNC_MATMUL_TRANSPOSED, + matmul_args + ); + + CoreTaskArgs activation_args; + activation_args.add_input(matmul.get_ref(0)); + activation_args.add_output(output); + rt_submit_aiv_task(FUNC_ACTIVATION, activation_args); +} + +void submit_layer(const CoreTaskArgs &args) { + rt_submit_graph(&graph_function, args, /*variant=*/0); +} +``` + +The function pointer is the default Graph identity. Trailing integral, +`float`, `double`, and `bool` construction parameters are forwarded to the +Graph function and hashed by value into the cache key. They are separate from +execution scalars in `CoreTaskArgs`: changing a construction parameter selects a +different Definition rather than patching an existing one. + +An explicit identity is available for call sites that need a stable name: + +```cpp +rt_submit_graph( + GRAPH_KEY("qwen_decoder_layer_v1"), + &graph_function, + args, + /*variant=*/0 +); +``` + +An explicit `GRAPH_KEY` must be unique for every distinct Graph function in an +orchestration callable. The explicit-key overload deliberately excludes the +Graph function pointer from the cache identity so the key remains stable; using +the same key for different functions can select the wrong recorded topology. + +There are no public `GraphArgs`, `GraphBindings`, `Patch`, or `ScalarRef` +types. The boundary is represented by `CoreTaskArgs`. + +Boundary scalars are pass-through bindings. Forward them directly with +`node_args.add_scalar(args.scalar(i))` or `copy_scalars_from(args, i, count)` +so recording can retain their source indices. + +Ordinary C++ value transformations do not retain boundary provenance. Both +`node_args.add_scalar(args.scalar(i) + 1)` and copying `args.scalar(i)` into a +local arithmetic variable before calling `add_scalar` produce an ordinary +static node scalar. That value is stored in the Definition, and later cache +hits reuse the first invocation's value without a warning. The runtime cannot +distinguish such a derived value from an intentional static literal after the +C++ expression has produced a plain arithmetic value. Compute the derived value +before constructing the Graph boundary and pass it as another boundary scalar, +perform the transformation in a kernel, or use a construction parameter when +the value changes the Graph structure. + +Access through a non-const `scalar()` invalidates inherited boundary provenance +conservatively, because returning a mutable reference cannot distinguish a read +from a later write. A Graph containing such an invalidated binding is not +cached, which prevents replay from silently replacing the transformed value +with the unmodified boundary value. + +## Supported dynamic and static data + +- Boundary ChipTensor addresses may change for every invocation. +- Boundary scalar values may change for every invocation. Their count is fixed + by the recorded boundary contract. Unused boundary scalars are allowed and do + not create internal scalar patches. +- A Graph boundary contains at least one ChipTensor. +- Construction parameters are part of Graph identity and may control the + function's task count, kernel selection, or other structural choices. +- Boundary ChipTensor shape, stride, dtype, size, direction, contiguity, and alias + partition must match the first invocation. +- Internal task scalars with no boundary source are fixed Definition data. +- Boundary storage is caller-owned. `INPUT`, `INOUT`, `OUTPUT_EXISTING`, and + `NO_DEP` are supported. A boundary `TensorCreateInfo` tagged `OUTPUT` is not. +- Early-resolve hints apply while recording the first invocation. Replayed + internal nodes use the saved completion topology without the hint. +- A recorded task may depend on a Graph-external producer when that producer + is the creator of a boundary ChipTensor. The outer Graph owns that dependency on + replay; arbitrary cross-boundary explicit dependencies remain unsupported. + +Structural or alias mismatch logs a warning and executes the Graph function +normally for that invocation. It never reuses heap offsets recorded for a +different shape. Debug builds also assert at these unsupported boundaries so +development catches a violated fixed-shape contract immediately; the ordinary +path remains the defensive release-build behavior. + +## Qwen decoder-layer example + +The upper layer packages all ChipTensor I/O in `CoreTaskArgs`; the wrapper has no +separate `hidden`, `weight`, or `output` parameters: + +```cpp +void qwen_decoder_layer(const CoreTaskArgs &args) { + const ChipTensor &hidden = args.tensor(0).ref(); + const ChipTensor &attention_weight = args.tensor(1).ref(); + const ChipTensor &mlp_weight = args.tensor(2).ref(); + const ChipTensor &output = args.tensor(3).ref(); + + const std::array hidden_shape{hidden.shapes[0]}; + TensorCreateInfo attention_out( + hidden_shape.data(), static_cast(hidden_shape.size()), hidden.dtype + ); + + CoreTaskArgs attention_args; + attention_args.add_input(hidden, attention_weight); + attention_args.add_output(attention_out); + attention_args.copy_scalars_from(args, 0, 1); // dynamic token position + TaskOutputTensors attention = + rt_submit_aic_task(FUNC_ATTENTION, attention_args); + + MixedKernels mlp; + mlp.aic_kernel_id = FUNC_MLP_AIC; + mlp.aiv0_kernel_id = FUNC_MLP_AIV; + + CoreTaskArgs mlp_args; + mlp_args.add_input(attention.get_ref(0), mlp_weight); + mlp_args.add_output(output); + rt_submit_task(mlp, mlp_args); +} + +void submit_qwen_decoder_layer(const CoreTaskArgs &args) { + rt_submit_graph(&qwen_decoder_layer, args); +} + +void decode_three_layers( + const std::array &hidden, + const std::array &attention_weight, + const std::array &mlp_weight, + const std::array &output, + const std::array &token_position +) { + for (std::size_t layer = 0; layer < hidden.size(); ++layer) { + CoreTaskArgs args; + args.add_input( + hidden[layer], + attention_weight[layer], + mlp_weight[layer] + ); + args.add_output(output[layer]); + args.add_scalar(token_position[layer]); + submit_qwen_decoder_layer(args); + } +} +``` + +All three layers submit one Graph task each: the first records the sub-DAG off +the ring and emits its Graph task, layers two and three replay the cached +Definition when their ChipTensor metadata and boundary scalar count match. Each +invocation patches the current layer's `token_position`; it is a dynamic +boundary scalar refreshed on every submission and is not part of the Graph key. + +## Definition + +Recording uses host-only C++ state: + +- `std::vector` for nodes, tensors, scalars, fanins, and pending uploads; +- `std::unordered_map` for the per-run Definition cache; +- `std::unique_ptr` for the active recording. + +The cache stores at most 16 Definitions and allocates each entry to its actual +serialized size. No fixed maximum-size recording array is copied on a cache +hit. + +At `graph_end`, recording is compacted into one contiguous, pointer-free POD +Definition. It contains: + +- node order and AIC/AIV/MIX/SPMD kernel metadata; +- `root_indices` plus both directions of the immutable topology: + fanin CSR and fanout CSR; +- one packed-heap offset per node; +- each node's ChipTensor source: + `BOUNDARY_EXACT`, `BOUNDARY_VIEW`, `INTERNAL`, or `OWN_OUTPUT`; +- fixed scalar values plus boundary-scalar source indices; +- fixed boundary signatures and alias representatives. + +The header also carries a content hash of the complete Definition image. The +device execution pool requires this hash, the Graph key, and the node count to +all match before reusing a resident Definition. A new run may record different +metadata under the same function identity, so key-only reuse is not safe. + +All references are 32-bit offsets from the Definition base. Cross-boundary +Tensors use the fixed-width `GraphTensor` wire POD rather than the +64-byte-aligned C++ `ChipTensor` object. The upload is therefore one contiguous +copy with no raw Host pointers and no relocation pass. + +Before materialization, the Scheduler recomputes the Definition content hash +and validates section ranges, topology indices, node heap offsets, the outer +heap extent, ChipTensor metadata, and ChipTensor-source bounds. Invalid wire data is +rejected before an offset participates in pointer arithmetic. + +There is no cache schema version. The cache is per run and starts empty, so a +persistent-format version would currently have no effect. + +## Cache hit and memory + +For a cache hit, the Host Orchestrator: + +1. validates the fixed boundary contract; +2. reserves one task-window slot; +3. reserves one heap block large enough for every internal intermediate; +4. computes only external fanin and boundary tensormap effects; +5. emits one outer `GRAPH` task; +6. stages the exact-size POD submission image for upload after orchestration; +7. asks the host runtime for an aligned execution block sized from the recorded + node count, Tensor-address and scalar patch capacities, and Definition + bytes, then writes that device address into the submission wire image. + +Internal nodes consume no ring task-window slots. Their descriptor, payload, +and slot state are built in host-owned GM. The runtime retains one grow-only +block per `(pipeline slot, Graph key, occurrence index)`: repeated runs on the +same slot reuse the allocation, repeated uses of one key within a run receive +distinct blocks, and the two pipeline slots never share an active block. Every +allocation goes through the Worker's tracked `MemoryAllocator`, contributes to +`committed_device_memory()`, and is released when the Worker is finalized. + +The `GraphSubmission` wire POD carries the aligned device address and usable +byte capacity explicitly. The Scheduler validates both before placement- +constructing `GraphExecution`; it never allocates execution storage from the +AICPU process heap. A block whose prior Definition key and content hash match +retains the local Definition, static node fields, and the Tensor-address and +scalar patch tables generated during its first materialization. That +graph-affine replay skips +topology binding, per-node count/offset validation, tensor-source +classification, tensor wire validation, static field stores, and static scalar +copies. It refreshes only task IDs, packed-buffer bases, boundary/internal +tensor addresses, boundary scalar bindings, scheduling state, dispatch +atomics, and wake registrations. + +The retained blocks are addressed directly by `(pipeline slot, Graph key, +occurrence index)`. Occurrence numbering restarts deterministically for every +run, so repeated layers map back to the same block in their pipeline slot; +affinity does not depend on a recycler selecting a recently freed block. + +## Scheduler flow + +Host orchestration builds the complete task image before device execution. At +the end of orchestration, the Host uploads every exact-size Graph POD image, +relocates the task and payload pointers in the shared-memory image, copies the +complete shared-memory/runtime-arena image to the device, and then launches the +resident Scheduler. + +All AICPU threads classify disjoint slices of the completed task window behind +one startup barrier. A Graph task enters preparation and external-fanin +classification during that scan, so Graph execution is interleaved with other +ready tasks at the same scheduling level once the Scheduler starts. + +This design does not overlap orchestration and scheduling within one run. +Prepared-successor pipelining can overlap preparation of run N+1 with device +execution of run N, while Graph cache hits reduce repeated orchestration work +inside a run. + +A Graph is placed in two independent control flows: + +- `graph_prepare_queue`: materialize the saved nodes even while external fanin + is still pending; +- `graph_ready_queue`: signal that the outer Graph's external fanin is ready. + +Core-owning Scheduler threads pop at most one item from each queue per loop. A +prepare call expands at most four nodes and requeues unfinished work, +interleaving Graph expansion with normal scheduling. + +Preparation and external readiness set two bits in one atomic activation gate. +Whichever operation sets the second bit activates the saved root nodes exactly +once. + +Internal dependency readiness borrows the completion-state polling idea, but +dependency wiring remains an Orchestrator responsibility: + +- recording constructs both fanin and fanout CSR in the immutable Definition; +- first materialization builds static runnable node state plus compact Tensor + address and scalar patch tables; affine replay applies those tables and + resets only dynamic runnable state; +- materialization registers each non-root on one producer selected from its + saved fanin CSR; +- a node's release/acquire `task_state` is its Graph-local completion flag, so + internal nodes need neither ring completion flags nor task-window slots; +- producer completion closes and drains only its current wake-list rather than + traversing the saved fanout CSR; +- a woken consumer scans its saved fanin CSR and either enters its shape queue + or registers on the next incomplete producer; +- `WAKE_LIST_SENTINEL` closes the completion/registration race: a failed + registration observes completion and immediately rescans. + +The runtime wake-list registration is a transient polling subscription, not +dependency discovery or Graph rewiring. Fanout CSR remains in the Definition +as part of the complete recorded topology and for DFX, but readiness does not +walk it. + +```text +outer GRAPH + -> activate root_indices[] + -> producer completion drains its current wake-list + -> each waiter polls saved fanin completion state + -> ready waiter enters its ordinary shape queue + or registers on another incomplete producer + -> final internal completion completes the outer GRAPH +``` + +Internal nodes count as zero outer ring tasks. The final node completes +the one outer Graph task, publishes the outer ring completion flag, wakes +external consumers, and contributes one to the host-visible completion count. + +Localization or materialization failure is fail-fast: the Scheduler latches an +error instead of leaving an already-submitted outer Graph unable to complete. + +## Current unsupported cases + +These cases assert in debug builds and execute through the ordinary path in a +release build: + +- an empty Graph boundary; +- variable ChipTensor shape or metadata; +- changed boundary aliasing; +- runtime-allocated boundary outputs; +- nested Graph recording; +- dispatch predicates; +- cross-boundary explicit dependencies that are not represented by a boundary + ChipTensor's creator; +- an unclassifiable internal ChipTensor source; +- a boundary-derived scalar accessed through mutable `scalar()`; +- more than 16 Definitions, 1024 internal nodes, or 32 boundary Tensors; +- insufficient task-window or heap capacity detected before outer submission. + +An AICPU execution-pool or materialization failure happens after the outer +Graph has already been submitted. It therefore latches a Scheduler fatal error +instead of falling back; leaving the outer task pending would otherwise wedge +completion. + +Explicit dependencies between recorded internal nodes are preserved when they +are otherwise supported; ordinary ChipTensor dependencies are always preserved. + +## DFX + +With L2 swimlane level 4: + +- `Graph Execution` spans an outer Graph execution; +- `AICPU Scheduler` shows bounded `graph_prepare` slices separately from normal + dispatch; +- existing Scheduler and Worker lanes show the expanded internal tasks. + +The scene coverage under `tests/st/{a2a3,a5}/host_build_graph/graph_execution` +includes AIV fanin/fanout DAGs, architecture-native AIC/AIV decoder-style DAGs, +and three-slot multi-block MIX/SPMD Graphs. Every scene invokes the same fixed +Graph three times: one recording execution followed by two outer-Graph +submissions. The full-model example at +`examples/a2a3/host_build_graph/qwen3_14b_decode` records one Qwen3-14B decoder +layer and replays its Definition for the remaining 39 layers. diff --git a/src/common/host_build_graph/graph_cache.h b/src/common/host_build_graph/graph_cache.h new file mode 100644 index 0000000000..8455b25b1c --- /dev/null +++ b/src/common/host_build_graph/graph_cache.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include +#include + +#include + +#include "pto_task_id.h" +#include "pto_types.h" + +inline constexpr uint32_t GRAPH_MAX_TENSOR_ARGS = 32; + +struct GraphScopeResult { + bool execute_block{true}; + bool recording{false}; + PTO2TaskId task_id{PTO2TaskId::invalid()}; +}; + +using GraphSubmitResult = GraphScopeResult; + +constexpr uint64_t graph_hash_byte(uint64_t h, uint8_t b) { return (h ^ static_cast(b)) * 1099511628211ULL; } + +inline uint64_t graph_hash_bytes(uint64_t h, const void *data, size_t bytes) { + const auto *p = static_cast(data); + for (size_t i = 0; i < bytes; ++i) { + h = graph_hash_byte(h, p[i]); + } + return h; +} + +constexpr uint64_t graph_const_hash_impl(const char *s, uint64_t h) { + return (*s == '\0') ? h : graph_const_hash_impl(s + 1, graph_hash_byte(h, static_cast(*s))); +} + +constexpr uint64_t GRAPH_KEY(const char *s) { return graph_const_hash_impl(s, 1469598103934665603ULL); } + +inline bool rt_graph_args_cacheable(const CoreTaskArgs &args) { + if (args.has_error || args.tensor_count() <= 0 || + args.tensor_count() > static_cast(GRAPH_MAX_TENSOR_ARGS)) { + return false; + } + for (int32_t i = 0; i < args.tensor_count(); ++i) { + // A Graph boundary is caller-owned storage. Runtime-allocated + // TensorCreateInfo outputs remain on the ordinary submit path. + if (args.tag(i) == TensorArgType::OUTPUT) return false; + } + return true; +} + +inline uint64_t rt_graph_make_key(uint64_t graph_id) { return graph_id; } + +template +inline uint64_t graph_hash_config_value(uint64_t hash, T value) { + using Value = std::remove_cv_t>; + static_assert( + std::is_integral_v || std::is_same_v || std::is_same_v, + "Graph construction parameters must be integral, float, or double values" + ); + constexpr uint8_t category = std::is_same_v ? 1 : + std::is_integral_v ? (std::is_signed_v ? 2 : 3) : + 4; + constexpr uint8_t width = sizeof(Value); + hash = graph_hash_byte(hash, category); + hash = graph_hash_byte(hash, width); + return graph_hash_bytes(hash, &value, sizeof(value)); +} + +template +inline uint64_t rt_graph_make_key(uint64_t graph_id, Config... config) { + uint64_t hash = graph_hash_bytes(1469598103934665603ULL, &graph_id, sizeof(graph_id)); + const uint32_t count = sizeof...(Config); + hash = graph_hash_bytes(hash, &count, sizeof(count)); + ((hash = graph_hash_config_value(hash, config)), ...); + return hash; +} diff --git a/src/a2a3/runtime/host_build_graph/runtime/scheduler/graph_execution.cpp b/src/common/host_build_graph/graph_execution.cpp similarity index 100% rename from src/a2a3/runtime/host_build_graph/runtime/scheduler/graph_execution.cpp rename to src/common/host_build_graph/graph_execution.cpp diff --git a/src/common/host_build_graph/graph_execution.h b/src/common/host_build_graph/graph_execution.h new file mode 100644 index 0000000000..9c3390f432 --- /dev/null +++ b/src/common/host_build_graph/graph_execution.h @@ -0,0 +1,475 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include +#include + +#include +#include + +#include "pto_runtime2_types.h" +#include "tensor.h" + +inline constexpr uint32_t GRAPH_MAX_NODES = 1024; +inline constexpr int32_t GRAPH_MATERIALIZE_SLICE_NODES = 4; + +enum class GraphTensorSource : uint8_t { + BOUNDARY_EXACT = 0, + BOUNDARY_VIEW = 1, + INTERNAL = 2, + OWN_OUTPUT = 3, +}; + +// Wire representation of ChipTensor. ChipTensor itself is a host/runtime C++ type with +// 64-byte alignment and helper methods; placing it inside vector +// would not guarantee that alignment. Keep the boundary image C-compatible and +// copy only semantic fields into this naturally 8-byte-aligned POD. +struct GraphTensor { + uint64_t buffer_addr; + uint64_t buffer_size; + uint64_t owner_task_id; + uint64_t start_offset; + uint64_t extent_elem; + int32_t version; + uint32_t shapes[MAX_TENSOR_DIMS]; + uint32_t strides[MAX_TENSOR_DIMS]; + uint8_t ndims; + uint8_t dtype; + uint8_t manual_dep; + uint8_t is_contiguous; + uint8_t address_space; + uint8_t reserved[3]; +}; + +// Everything from GraphTensorSourceRef through GraphSubmission is copied +// across the host-device boundary. Keep it pointer-free, fixed-width and +// position-independent: every reference is an offset from its owning header. +struct GraphTensorSourceRef { + uint8_t source; + uint8_t reserved; + uint16_t source_index; + uint32_t reserved2; + uint64_t packed_offset; +}; + +enum class GraphScalarSource : uint8_t { + STATIC_VALUE = 0, + BOUNDARY = 1, +}; + +struct GraphScalarSourceRef { + uint16_t source_index; + uint8_t source; + uint8_t reserved; +}; + +struct GraphNodeDefinition { + int32_t kernel_id[PTO2_SUBTASK_SLOT_COUNT]; + uint8_t active_mask; + uint8_t task_attrs; + int16_t logical_block_num; + int16_t total_required_subtasks; + uint16_t reserved; + int32_t tensor_count; + int32_t scalar_count; + int32_t total_output_size; + uint32_t tensor_offset; + uint32_t scalar_offset; + ArgsDumpTaskMetadata dump_metadata; +}; + +struct GraphBoundarySignature { + uint64_t buffer_size; + uint32_t shapes[MAX_TENSOR_DIMS]; + uint32_t strides[MAX_TENSOR_DIMS]; + uint16_t alias_rep; + uint8_t ndims; + uint8_t dtype; + uint8_t tag; + uint8_t manual_dep; + uint8_t is_contiguous; + uint8_t reserved; +}; + +struct GraphDefinition { + uint64_t full_key; + uint64_t content_hash; + uint64_t required_heap; + uint32_t total_bytes; + uint32_t task_count; + uint32_t edge_count; + uint32_t root_count; + uint32_t boundary_count; + uint32_t boundary_scalar_count; + uint32_t tensor_arg_count; + uint32_t scalar_arg_count; + uint32_t off_fanout_offsets; + uint32_t off_fanout_indices; + uint32_t off_fanin_offsets; + uint32_t off_fanin_indices; + uint32_t off_root_indices; + uint32_t off_node_offsets; + uint32_t off_nodes; + uint32_t off_tensors; + uint32_t off_tensor_sources; + uint32_t off_scalars; + uint32_t off_scalar_sources; + uint32_t off_boundary_signatures; +}; + +struct GraphSubmission { + uint64_t graph_key; + uint64_t execution_storage; + uint64_t execution_storage_bytes; + uint64_t local_execution; + uint32_t activation_gate; + uint32_t total_bytes; + uint32_t definition_offset; + uint32_t tensors_offset; + uint32_t tensor_count; + uint32_t scalars_offset; + uint32_t scalar_count; +}; + +static_assert(std::is_trivially_copyable_v); +static_assert(std::is_standard_layout_v); +static_assert(std::is_trivially_copyable_v); +static_assert(std::is_standard_layout_v); +static_assert(std::is_trivially_copyable_v); +static_assert(std::is_standard_layout_v); +static_assert(std::is_trivially_copyable_v); +static_assert(std::is_standard_layout_v); +static_assert(std::is_trivially_copyable_v); +static_assert(std::is_standard_layout_v); +static_assert(std::is_trivially_copyable_v); +static_assert(std::is_standard_layout_v); +static_assert(std::is_trivially_copyable_v); +static_assert(std::is_standard_layout_v); + +inline GraphTensor graph_tensor_pack(const ChipTensor &tensor) { + GraphTensor packed{}; + packed.buffer_addr = tensor.buffer.addr; + packed.buffer_size = tensor.buffer.size; + packed.owner_task_id = tensor.owner_task_id.raw; + packed.start_offset = tensor.start_offset; + packed.extent_elem = tensor.extent_elem_cache; + packed.version = tensor.version; + for (uint32_t i = 0; i < tensor.ndims; ++i) { + packed.shapes[i] = tensor.shapes[i]; + packed.strides[i] = tensor.strides[i]; + } + packed.ndims = static_cast(tensor.ndims); + packed.dtype = static_cast(tensor.dtype); + packed.manual_dep = tensor.manual_dep ? 1 : 0; + packed.is_contiguous = tensor.is_contiguous ? 1 : 0; + packed.address_space = static_cast(tensor.address_space); + return packed; +} + +inline void graph_tensor_unpack(const GraphTensor &packed, ChipTensor *tensor) { + tensor->buffer = PTOBufferHandle{packed.buffer_addr, packed.buffer_size}; + tensor->owner_task_id = PTO2TaskId{packed.owner_task_id}; + tensor->start_offset = packed.start_offset; + tensor->extent_elem_cache = packed.extent_elem; + tensor->version = packed.version; + tensor->ndims = packed.ndims; + tensor->dtype = static_cast(packed.dtype); + tensor->manual_dep = packed.manual_dep != 0; + tensor->is_contiguous = packed.is_contiguous != 0; + tensor->address_space = static_cast(packed.address_space); + for (uint32_t i = 0; i < MAX_TENSOR_DIMS; ++i) { + tensor->shapes[i] = packed.shapes[i]; + tensor->strides[i] = packed.strides[i]; + } + for (uint8_t &byte : tensor->_pad_cl2) + byte = 0; +} + +inline bool graph_tensor_wire_valid(const GraphTensor &tensor) { + if (tensor.buffer_addr == 0 || tensor.ndims == 0 || tensor.ndims > MAX_TENSOR_DIMS || + tensor.dtype >= static_cast(DataType::DATA_TYPE_NUM) || tensor.manual_dep > 1 || + tensor.is_contiguous > 1 || tensor.address_space > 1) { + return false; + } + + uint64_t extent = 1; + uint64_t expected_stride = 1; + bool contiguous = true; + for (int32_t i = static_cast(tensor.ndims) - 1; i >= 0; --i) { + const uint64_t shape = tensor.shapes[i]; + const uint64_t stride = tensor.strides[i]; + if (shape == 0 || stride == 0) return false; + contiguous &= stride == expected_stride; + if (shape - 1 > (UINT64_MAX - extent) / stride || expected_stride > UINT64_MAX / shape) return false; + extent += (shape - 1) * stride; + expected_stride *= shape; + } + if (extent != tensor.extent_elem || contiguous != (tensor.is_contiguous != 0)) return false; + + const uint64_t element_size = get_element_size(static_cast(tensor.dtype)); + const uint64_t buffer_elements = tensor.buffer_size / element_size; + return tensor.start_offset <= buffer_elements && tensor.extent_elem <= buffer_elements - tensor.start_offset; +} + +template +inline const T *graph_definition_array(const GraphDefinition &definition, uint32_t offset, uint32_t count) { + if (offset == 0 || offset > definition.total_bytes || offset % alignof(T) != 0) return nullptr; + const size_t remaining = static_cast(definition.total_bytes - offset); + if (count > remaining / sizeof(T)) return nullptr; + return reinterpret_cast(reinterpret_cast(&definition) + offset); +} + +template +inline const T *graph_definition_ptr(const GraphDefinition &definition, uint32_t offset) { + return graph_definition_array(definition, offset, 1); +} + +inline GraphSubmission *graph_submission_from_slot(PTO2TaskSlotState &slot) { + return slot.task_kind == TaskKind::GRAPH ? static_cast(slot.graph_context) : nullptr; +} + +inline const GraphDefinition *graph_submission_definition(const GraphSubmission &submission) { + if (submission.definition_offset == 0 || submission.definition_offset % alignof(GraphDefinition) != 0 || + submission.definition_offset > submission.total_bytes || + sizeof(GraphDefinition) > submission.total_bytes - submission.definition_offset) { + return nullptr; + } + const auto *definition = reinterpret_cast( + reinterpret_cast(&submission) + submission.definition_offset + ); + if (definition->total_bytes < sizeof(GraphDefinition) || + definition->total_bytes > submission.total_bytes - submission.definition_offset) { + return nullptr; + } + return definition; +} + +inline bool graph_submission_wire_size_valid(const GraphSubmission &submission, size_t available_bytes) { + return available_bytes >= sizeof(GraphSubmission) && submission.total_bytes == available_bytes; +} + +inline const GraphTensor *graph_submission_tensors(const GraphSubmission &submission) { + if (submission.tensors_offset == 0 || submission.tensors_offset % alignof(GraphTensor) != 0 || + submission.tensors_offset > submission.total_bytes || + submission.tensor_count > (submission.total_bytes - submission.tensors_offset) / sizeof(GraphTensor)) { + return nullptr; + } + return reinterpret_cast( + reinterpret_cast(&submission) + submission.tensors_offset + ); +} + +inline const uint64_t *graph_submission_scalars(const GraphSubmission &submission) { + if (submission.scalar_count == 0) return nullptr; + if (submission.scalars_offset == 0 || submission.scalars_offset % alignof(uint64_t) != 0 || + submission.scalars_offset > submission.total_bytes || + submission.scalar_count > (submission.total_bytes - submission.scalars_offset) / sizeof(uint64_t)) { + return nullptr; + } + return reinterpret_cast( + reinterpret_cast(&submission) + submission.scalars_offset + ); +} + +enum class GraphExecutionState : uint8_t { + SUBMITTED = 0, + MATERIALIZING = 1, + PREPARED = 2, + ACTIVE = 3, + COMPLETED = 4, +}; + +enum class GraphMaterializeResult : uint8_t { + INVALID = 0, + BUSY = 1, + PENDING = 2, + PREPARED = 3, +}; + +enum class GraphTensorAddressSource : uint8_t { + BOUNDARY = 0, + INTERNAL = 1, +}; + +// Precomputed on the first materialization and retained next to the node +// storage. Affine replay walks this compact POD instead of re-reading and +// classifying GraphTensorSourceRef entries from the Definition. +struct GraphTensorAddressPatch { + uint64_t address_offset; + uint16_t source_index; + uint8_t source; + uint8_t reserved[5]; +}; + +static_assert(std::is_trivially_copyable_v); +static_assert(std::is_standard_layout_v); +static_assert(sizeof(GraphTensorAddressPatch) == 16); + +struct GraphScalarPatch { + uint16_t node_index; + uint8_t node_scalar_index; + uint8_t boundary_scalar_index; +}; + +static_assert(std::is_trivially_copyable_v); +static_assert(std::is_standard_layout_v); +static_assert(sizeof(GraphScalarPatch) == 4); +static_assert(GRAPH_MAX_NODES <= UINT16_MAX); +static_assert(MAX_SCALAR_ARGS <= UINT8_MAX); + +struct alignas(64) GraphNodeStorage { + PTO2TaskDescriptor task; + PTO2TaskPayload payload; + PTO2TaskSlotState slot; +}; + +inline constexpr uint64_t GRAPH_EXECUTION_STORAGE_MAGIC = 0x4752415048455845ULL; +inline constexpr uint64_t GRAPH_EXECUTION_INITIALIZING = 1; + +struct GraphExecution { + uint64_t storage_magic{0}; + std::atomic state{GraphExecutionState::SUBMITTED}; + std::atomic materialize_busy{0}; + std::atomic remaining_nodes{0}; + std::atomic retired_nodes{0}; + // Incremental activation: nodes in [0, published_nodes) are fully + // materialized and registered, so a route pass may consider them. route_cursor + // is the next such node index a route pass will claim; roots below it have + // been pushed to the ready queue exactly once. Both advance monotonically and + // reset per (re)submission. + std::atomic published_nodes{0}; + std::atomic route_cursor{0}; + int32_t node_count{0}; + int32_t node_capacity{0}; + int32_t materialized_nodes{0}; + int32_t materialized_node_count{0}; + int32_t constructed_nodes{0}; + uint32_t tensor_patch_capacity{0}; + uint32_t materialized_tensor_patches{0}; + uint32_t materialized_tensor_patch_count{0}; + uint32_t scalar_patch_capacity{0}; + uint32_t materialized_scalar_patches{0}; + uint32_t materialized_scalar_patch_count{0}; + size_t allocation_bytes{0}; + size_t definition_capacity{0}; + uint64_t graph_key{0}; + uint64_t definition_hash{0}; + uint64_t materialized_graph_key{0}; + uint64_t materialized_definition_hash{0}; + uintptr_t materialized_outer_base{0}; + bool definition_affine_reuse{false}; + PTO2TaskSlotState *outer_slot{nullptr}; + GraphNodeStorage *nodes{nullptr}; + GraphNodeStorage *node_storage{nullptr}; + GraphTensorAddressPatch *tensor_patches{nullptr}; + GraphScalarPatch *scalar_patches{nullptr}; + void *definition_storage{nullptr}; + const GraphDefinition *definition{nullptr}; + const uint32_t *fanin_offsets{nullptr}; + const uint16_t *fanin_indices{nullptr}; + const GraphTensor *boundary_tensors{nullptr}; + uint32_t boundary_tensor_count{0}; + const uint64_t *boundary_scalars{nullptr}; + uint32_t boundary_scalar_count{0}; +}; + +static_assert(offsetof(GraphExecution, storage_magic) == 0); +static_assert(sizeof(GraphExecution::storage_magic) == sizeof(uint64_t)); +static_assert(std::is_trivially_destructible_v); +static_assert(std::is_trivially_destructible_v); + +inline bool graph_execution_storage_layout( + int32_t node_capacity, uint32_t tensor_patch_capacity, uint32_t scalar_patch_capacity, size_t definition_capacity, + size_t *nodes_offset, size_t *tensor_patches_offset, size_t *scalar_patches_offset, size_t *definition_offset, + size_t *storage_bytes +) { + if (nodes_offset == nullptr || tensor_patches_offset == nullptr || scalar_patches_offset == nullptr || + definition_offset == nullptr || storage_bytes == nullptr || node_capacity <= 0 || + static_cast(node_capacity) > SIZE_MAX / sizeof(GraphNodeStorage) || + tensor_patch_capacity > GRAPH_MAX_NODES * MAX_TENSOR_ARGS || + scalar_patch_capacity > GRAPH_MAX_NODES * MAX_SCALAR_ARGS) { + return false; + } + auto checked_align_up = [](size_t value, size_t alignment, size_t *result) { + if (alignment == 0 || value > SIZE_MAX - (alignment - 1)) return false; + *result = (value + alignment - 1) & ~(alignment - 1); + return true; + }; + const size_t nodes_bytes = static_cast(node_capacity) * sizeof(GraphNodeStorage); + const size_t tensor_patches_bytes = static_cast(tensor_patch_capacity) * sizeof(GraphTensorAddressPatch); + const size_t scalar_patches_bytes = static_cast(scalar_patch_capacity) * sizeof(GraphScalarPatch); + if (!checked_align_up(sizeof(GraphExecution), alignof(GraphNodeStorage), nodes_offset) || + *nodes_offset > SIZE_MAX - nodes_bytes || + !checked_align_up(*nodes_offset + nodes_bytes, alignof(GraphTensorAddressPatch), tensor_patches_offset) || + *tensor_patches_offset > SIZE_MAX - tensor_patches_bytes || + !checked_align_up( + *tensor_patches_offset + tensor_patches_bytes, alignof(GraphScalarPatch), scalar_patches_offset + ) || + *scalar_patches_offset > SIZE_MAX - scalar_patches_bytes || + !checked_align_up(*scalar_patches_offset + scalar_patches_bytes, alignof(GraphDefinition), definition_offset) || + *definition_offset > SIZE_MAX - definition_capacity) { + return false; + } + return checked_align_up(*definition_offset + definition_capacity, alignof(GraphNodeStorage), storage_bytes); +} + +inline bool graph_execution_storage_bytes( + int32_t node_capacity, uint32_t tensor_patch_capacity, uint32_t scalar_patch_capacity, size_t definition_capacity, + size_t *storage_bytes +) { + size_t nodes_offset = 0; + size_t tensor_patches_offset = 0; + size_t scalar_patches_offset = 0; + size_t definition_offset = 0; + return graph_execution_storage_layout( + node_capacity, tensor_patch_capacity, scalar_patch_capacity, definition_capacity, &nodes_offset, + &tensor_patches_offset, &scalar_patches_offset, &definition_offset, storage_bytes + ); +} + +GraphExecution *graph_execution_localize(PTO2TaskSlotState &outer_slot); +GraphMaterializeResult graph_execution_materialize_slice( + PTO2TaskSlotState &outer_slot, GraphExecution &execution, int32_t max_nodes, int32_t *nodes_materialized = nullptr +); + +inline GraphExecution *graph_execution_from_slot(PTO2TaskSlotState &slot) { + return slot.task_kind == TaskKind::GRAPH_NODE ? static_cast(slot.graph_context) : nullptr; +} + +inline bool graph_execution_complete_node(GraphExecution &execution) { + return execution.remaining_nodes.fetch_sub(1, std::memory_order_acq_rel) == 1; +} + +inline void graph_execution_mark_completed(GraphExecution &execution) { + execution.state.store(GraphExecutionState::COMPLETED, std::memory_order_release); +} + +inline void graph_execution_retire_node(GraphExecution &execution) { + execution.retired_nodes.fetch_add(1, std::memory_order_release); +} + +inline bool graph_submission_signal(GraphSubmission &submission, uint32_t bit) { + constexpr uint32_t BOTH = 0x3; + uint32_t observed = __atomic_fetch_or(&submission.activation_gate, bit, __ATOMIC_ACQ_REL); + return observed != BOTH && (observed | bit) == BOTH; +} + +inline GraphExecution *graph_submission_local_execution(GraphSubmission &submission) { + uint64_t raw = __atomic_load_n(&submission.local_execution, __ATOMIC_ACQUIRE); + if (raw <= GRAPH_EXECUTION_INITIALIZING) return nullptr; + return reinterpret_cast(static_cast(raw)); +} + +inline bool graph_submission_execution_initializing(const GraphSubmission &submission) { + return __atomic_load_n(&submission.local_execution, __ATOMIC_ACQUIRE) == GRAPH_EXECUTION_INITIALIZING; +} diff --git a/src/common/host_build_graph/graph_host_state.h b/src/common/host_build_graph/graph_host_state.h new file mode 100644 index 0000000000..4d869b1c4d --- /dev/null +++ b/src/common/host_build_graph/graph_host_state.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include +#include +#include + +struct PTO2TaskSlotState; +struct GraphHostState; + +inline constexpr size_t GRAPH_MAX_DEFINITIONS = 16; + +struct GraphHostStateDeleter { + void operator()(GraphHostState *state) const noexcept; +}; + +using GraphHostStatePtr = std::unique_ptr; + +struct GraphHostUpload { + PTO2TaskSlotState *outer_slot; + std::byte *data; + size_t bytes; +}; + +GraphHostStatePtr make_graph_host_state(); +size_t graph_host_upload_count(const GraphHostState &state); +std::optional graph_host_upload(GraphHostState &state, size_t index); diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index ac2ae5facd..df8b8944de 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -848,13 +848,13 @@ target_sources(test_a5_hbg_submit_poison PRIVATE ${HBG_A5_RUNTIME_DIR}/shared/runtime.cpp ${CMAKE_SOURCE_DIR}/../../../src/common/platform/shared/aicpu/args_dump_aicpu.cpp ) -add_a2a3_hbg_runtime_test(test_graph_cache a2a3/test_graph_cache.cpp) +add_a2a3_hbg_runtime_test(test_graph_cache common/test_hbg_graph_cache.cpp) target_sources(test_graph_cache PRIVATE - ${CMAKE_SOURCE_DIR}/../../../src/a2a3/runtime/host_build_graph/runtime/scheduler/graph_execution.cpp + ${CMAKE_SOURCE_DIR}/../../../src/common/host_build_graph/graph_execution.cpp ) -add_a5_hbg_runtime_test(test_a5_graph_cache a5/test_graph_cache.cpp) +add_a5_hbg_runtime_test(test_a5_graph_cache common/test_hbg_graph_cache.cpp) target_sources(test_a5_graph_cache PRIVATE - ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/host_build_graph/runtime/scheduler/graph_execution.cpp + ${CMAKE_SOURCE_DIR}/../../../src/common/host_build_graph/graph_execution.cpp ) # Incremental-activation tests init a real PTO2SchedulerState (ready queues), so # they link the scheduler + shared runtime out-of-line members. diff --git a/tests/ut/cpp/a5/test_graph_cache.cpp b/tests/ut/cpp/a5/test_graph_cache.cpp deleted file mode 100644 index 529f2c4bae..0000000000 --- a/tests/ut/cpp/a5/test_graph_cache.cpp +++ /dev/null @@ -1,486 +0,0 @@ -/* - * Copyright (c) PyPTO Contributors. - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of - * CANN Open Software License Agreement Version 2.0 (the "License"). - * Please refer to the License for details. You may not use this file except in compliance with the License. - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. - * See LICENSE in the root of the software repository for the full text of the License. - * ----------------------------------------------------------------------------------------------------------- - */ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "graph_cache.h" -#include "graph_execution.h" -#include "runtime_status/error_names.h" -#include "scheduler/pto_scheduler.h" - -namespace { - -template -uint32_t append_section(std::vector &image, const std::vector &values) { - if (values.empty()) return 0; - const size_t offset = PTO2_ALIGN_UP(image.size(), alignof(T)); - image.resize(offset + values.size() * sizeof(T)); - std::memcpy(image.data() + offset, values.data(), values.size() * sizeof(T)); - return static_cast(offset); -} - -GraphTensor make_test_tensor(uint64_t address) { - GraphTensor tensor{}; - tensor.buffer_addr = address; - tensor.buffer_size = 64; - tensor.extent_elem = 1; - tensor.shapes[0] = 1; - tensor.strides[0] = 1; - tensor.ndims = 1; - tensor.dtype = static_cast(DataType::FLOAT32); - tensor.is_contiguous = 1; - return tensor; -} - -std::vector make_test_definition(uint64_t graph_key, uint64_t boundary_address) { - std::vector image(sizeof(GraphDefinition)); - - std::vector fanin_offsets{0, 0, 1}; - std::vector fanin_indices{0}; - std::vector fanout_offsets{0, 1, 1}; - std::vector fanout_indices{1}; - std::vector roots{0}; - std::vector node_offsets{0, 64}; - std::vector nodes(2); - for (GraphNodeDefinition &node : nodes) { - std::fill(std::begin(node.kernel_id), std::end(node.kernel_id), INVALID_KERNEL_ID); - node.kernel_id[0] = 42; - node.active_mask = 1; - node.logical_block_num = 1; - node.total_required_subtasks = 1; - node.tensor_count = 1; - node.scalar_count = 1; - node.total_output_size = 64; - } - nodes[0].dump_metadata.dump_arg_mask = uint64_t{1} << 0; - nodes[0].dump_metadata.scalar_dtypes[0] = static_cast(DataType::FLOAT32); - nodes[1].dump_metadata.dump_arg_mask = uint64_t{1} << 1; - nodes[1].dump_metadata.scalar_dtypes[0] = static_cast(DataType::INT32); - nodes[1].tensor_offset = 1; - nodes[1].scalar_offset = 1; - std::vector tensors{make_test_tensor(boundary_address), make_test_tensor(boundary_address)}; - tensors[1].buffer_size = 32; - std::vector tensor_sources(2); - tensor_sources[0].source = static_cast(GraphTensorSource::BOUNDARY_EXACT); - tensor_sources[1].source = static_cast(GraphTensorSource::INTERNAL); - tensor_sources[1].packed_offset = 16; - std::vector scalars{0, 18}; - std::vector scalar_sources(2); - scalar_sources[0].source = static_cast(GraphScalarSource::BOUNDARY); - scalar_sources[1].source = static_cast(GraphScalarSource::STATIC_VALUE); - - GraphDefinition definition{}; - definition.full_key = graph_key; - definition.required_heap = 128; - definition.task_count = 2; - definition.edge_count = 1; - definition.root_count = 1; - definition.boundary_count = 1; - definition.boundary_scalar_count = 1; - definition.tensor_arg_count = 2; - definition.scalar_arg_count = 2; - definition.off_fanin_offsets = append_section(image, fanin_offsets); - definition.off_fanin_indices = append_section(image, fanin_indices); - definition.off_fanout_offsets = append_section(image, fanout_offsets); - definition.off_fanout_indices = append_section(image, fanout_indices); - definition.off_root_indices = append_section(image, roots); - definition.off_node_offsets = append_section(image, node_offsets); - definition.off_nodes = append_section(image, nodes); - definition.off_tensors = append_section(image, tensors); - definition.off_tensor_sources = append_section(image, tensor_sources); - definition.off_scalars = append_section(image, scalars); - definition.off_scalar_sources = append_section(image, scalar_sources); - definition.total_bytes = static_cast(image.size()); - std::memcpy(image.data(), &definition, sizeof(definition)); - - definition.content_hash = graph_hash_bytes(1469598103934665603ULL, image.data(), image.size()); - std::memcpy(image.data(), &definition, sizeof(definition)); - return image; -} - -std::vector make_test_submission( - uint64_t graph_key, uint64_t boundary_address, uint64_t boundary_scalar, uint64_t execution_storage, - size_t execution_storage_bytes -) { - const std::vector definition = make_test_definition(graph_key, boundary_address); - const size_t definition_offset = PTO2_ALIGN_UP(sizeof(GraphSubmission), alignof(GraphDefinition)); - const size_t tensors_offset = PTO2_ALIGN_UP(definition_offset + definition.size(), alignof(GraphTensor)); - const size_t scalars_offset = PTO2_ALIGN_UP(tensors_offset + sizeof(GraphTensor), alignof(uint64_t)); - std::vector image(scalars_offset + sizeof(uint64_t)); - std::memcpy(image.data() + definition_offset, definition.data(), definition.size()); - const GraphTensor boundary = make_test_tensor(boundary_address); - std::memcpy(image.data() + tensors_offset, &boundary, sizeof(boundary)); - std::memcpy(image.data() + scalars_offset, &boundary_scalar, sizeof(boundary_scalar)); - - GraphSubmission submission{}; - submission.graph_key = graph_key; - submission.execution_storage = execution_storage; - submission.execution_storage_bytes = execution_storage_bytes; - submission.total_bytes = static_cast(image.size()); - submission.definition_offset = static_cast(definition_offset); - submission.tensors_offset = static_cast(tensors_offset); - submission.tensor_count = 1; - submission.scalars_offset = static_cast(scalars_offset); - submission.scalar_count = 1; - std::memcpy(image.data(), &submission, sizeof(submission)); - return image; -} - -class AlignedStorage { -public: - explicit AlignedStorage(size_t bytes) : - bytes_(bytes) { - data_ = ::operator new(bytes, std::align_val_t(alignof(GraphNodeStorage))); - std::memset(data_, 0, bytes); - } - - ~AlignedStorage() { ::operator delete(data_, std::align_val_t(alignof(GraphNodeStorage))); } - - void *data() const { return data_; } - size_t size() const { return bytes_; } - -private: - void *data_{nullptr}; - size_t bytes_{0}; -}; - -} // namespace - -TEST(GraphCache, RejectsEmptyBoundary) { - CoreTaskArgs args; - - EXPECT_FALSE(rt_graph_args_cacheable(args)); -} - -TEST(GraphCache, AcceptsBoundaryScalars) { - std::array boundary{}; - const GraphTensor packed = make_test_tensor(reinterpret_cast(boundary.data())); - ChipTensor tensor{}; - graph_tensor_unpack(packed, &tensor); - - CoreTaskArgs args; - args.add_input(tensor); - args.add_scalar(uint32_t{17}); - - EXPECT_TRUE(rt_graph_args_cacheable(args)); -} - -TEST(GraphCache, ConfigValuesSelectDifferentDefinitions) { - constexpr uint64_t GRAPH_ID = 0x1234; - - EXPECT_NE(rt_graph_make_key(GRAPH_ID, 0), rt_graph_make_key(GRAPH_ID, 1)); - EXPECT_EQ(rt_graph_make_key(GRAPH_ID, 0), rt_graph_make_key(GRAPH_ID, 0)); -} - -TEST(GraphScalarProvenance, ForwardedScalarRetainsBoundarySource) { - uint32_t value = 17; - CoreTaskArgs boundary_args; - boundary_args.add_scalar(value, value); - boundary_args.anchor_scalar_sources(); - CoreTaskArgs forwarded_args; - forwarded_args.copy_scalars_from(boundary_args, 1, 1); - CoreTaskArgs node_args; - - node_args.copy_scalars_from(forwarded_args, 0, 1); - - EXPECT_EQ(node_args.scalar_source(0), static_cast(&std::as_const(boundary_args).scalar(1))); -} - -TEST(GraphScalarProvenance, MutableAccessInvalidatesForwardedSource) { - CoreTaskArgs boundary_args; - boundary_args.add_scalar(uint32_t{17}); - boundary_args.anchor_scalar_sources(); - CoreTaskArgs node_args; - node_args.copy_scalars_from(boundary_args, 0, 1); - ASSERT_NE(node_args.scalar_source(0), nullptr); - - node_args.scalar(0) = 18; - - EXPECT_EQ(node_args.scalar_source(0), nullptr); - EXPECT_EQ( - node_args.invalidated_scalar_source(0), static_cast(&std::as_const(boundary_args).scalar(0)) - ); -} - -TEST(GraphExecutionStorage, ComputesAlignedExactSize) { - constexpr int32_t NODE_COUNT = 7; - constexpr size_t DEFINITION_BYTES = 321; - constexpr uint32_t TENSOR_PATCH_COUNT = 11; - constexpr uint32_t SCALAR_PATCH_COUNT = 5; - size_t nodes_offset = 0; - size_t tensor_patches_offset = 0; - size_t scalar_patches_offset = 0; - size_t definition_offset = 0; - size_t storage_bytes = 0; - - ASSERT_TRUE(graph_execution_storage_layout( - NODE_COUNT, TENSOR_PATCH_COUNT, SCALAR_PATCH_COUNT, DEFINITION_BYTES, &nodes_offset, &tensor_patches_offset, - &scalar_patches_offset, &definition_offset, &storage_bytes - )); - EXPECT_EQ(nodes_offset % alignof(GraphNodeStorage), 0U); - EXPECT_EQ(tensor_patches_offset % alignof(GraphTensorAddressPatch), 0U); - EXPECT_EQ(scalar_patches_offset % alignof(GraphScalarPatch), 0U); - EXPECT_EQ(definition_offset % alignof(GraphDefinition), 0U); - EXPECT_GE(tensor_patches_offset, nodes_offset + NODE_COUNT * sizeof(GraphNodeStorage)); - EXPECT_GE(scalar_patches_offset, tensor_patches_offset + TENSOR_PATCH_COUNT * sizeof(GraphTensorAddressPatch)); - EXPECT_GE(definition_offset, scalar_patches_offset + SCALAR_PATCH_COUNT * sizeof(GraphScalarPatch)); - EXPECT_GE(storage_bytes, definition_offset + DEFINITION_BYTES); - EXPECT_EQ(storage_bytes % alignof(GraphNodeStorage), 0U); -} - -TEST(GraphExecutionStorage, RejectsInvalidCapacity) { - size_t storage_bytes = 0; - - EXPECT_FALSE(graph_execution_storage_bytes(0, 0, 0, sizeof(GraphDefinition), &storage_bytes)); - EXPECT_FALSE(graph_execution_storage_bytes(1, 0, 0, SIZE_MAX, &storage_bytes)); -} - -TEST(GraphExecutionReplay, AffineHitRefreshesOnlyDynamicFields) { - constexpr uint64_t GRAPH_KEY_VALUE = 0x1234; - std::array first_heap{}; - std::array second_heap{}; - std::array first_boundary{}; - std::array second_boundary{}; - - const std::vector definition = - make_test_definition(GRAPH_KEY_VALUE, reinterpret_cast(first_boundary.data())); - size_t execution_bytes = 0; - ASSERT_TRUE(graph_execution_storage_bytes(2, 2, 2, definition.size(), &execution_bytes)); - AlignedStorage execution_storage(execution_bytes); - std::vector submission_image = make_test_submission( - GRAPH_KEY_VALUE, reinterpret_cast(first_boundary.data()), 17, - reinterpret_cast(execution_storage.data()), execution_storage.size() - ); - auto &submission = *reinterpret_cast(submission_image.data()); - - PTO2TaskDescriptor outer_task{}; - outer_task.task_id = PTO2TaskId::make(1, 7); - outer_task.packed_buffer_base = first_heap.data(); - outer_task.packed_buffer_end = first_heap.data() + first_heap.size(); - PTO2TaskSlotState outer_slot{}; - outer_slot.task_kind = TaskKind::GRAPH; - outer_slot.task = &outer_task; - outer_slot.graph_context = &submission; - - GraphExecution *execution = graph_execution_localize(outer_slot); - ASSERT_NE(execution, nullptr); - EXPECT_EQ(graph_execution_materialize_slice(outer_slot, *execution, 2), GraphMaterializeResult::PREPARED); - GraphNodeStorage &node = execution->node_storage[0]; - ASSERT_EQ(node.payload.scalar_count, 1); - ASSERT_EQ(node.payload.tensor_count, 1); - EXPECT_EQ(node.payload.scalars[0], 17U); - EXPECT_EQ(execution->node_storage[1].payload.scalars[0], 18U); - EXPECT_EQ(node.payload.dump_metadata.dump_arg_mask, uint64_t{1} << 0); - EXPECT_EQ(node.payload.dump_metadata.scalar_dtypes[0], static_cast(DataType::FLOAT32)); - EXPECT_EQ(execution->node_storage[1].payload.dump_metadata.dump_arg_mask, uint64_t{1} << 1); - EXPECT_EQ(execution->node_storage[1].payload.dump_metadata.scalar_dtypes[0], static_cast(DataType::INT32)); - - graph_execution_mark_completed(*execution); - execution->retired_nodes.store(2, std::memory_order_release); - submission.local_execution = 0; - outer_task.task_id = PTO2TaskId::make(1, 8); - outer_task.packed_buffer_base = second_heap.data(); - outer_task.packed_buffer_end = second_heap.data() + second_heap.size(); - auto *boundary = reinterpret_cast(submission_image.data() + submission.tensors_offset); - boundary->buffer_addr = reinterpret_cast(second_boundary.data()); - auto *boundary_scalar = reinterpret_cast(submission_image.data() + submission.scalars_offset); - *boundary_scalar = 99; - - execution = graph_execution_localize(outer_slot); - ASSERT_NE(execution, nullptr); - ASSERT_TRUE(execution->definition_affine_reuse); - - // Write probes make an otherwise same-valued store observable. An affine - // replay must not touch these static fields; only the tensor address and - // per-run descriptor/scheduling state below are dynamic. - node.task.kernel_id[0] = 314; - node.slot.active_mask = ActiveMask(3); - node.payload.scalars[0] = 2718; - execution->node_storage[1].payload.scalars[0] = 31415; - node.payload.tensors[0].version = 1618; - node.slot.completed_subtasks.store(1, std::memory_order_relaxed); - node.payload.dispatch_fanin.store(1, std::memory_order_relaxed); - - EXPECT_EQ(graph_execution_materialize_slice(outer_slot, *execution, 2), GraphMaterializeResult::PREPARED); - EXPECT_EQ(node.task.kernel_id[0], 314); - EXPECT_EQ(node.slot.active_mask.raw(), 3); - EXPECT_EQ(node.payload.scalars[0], 99U); - EXPECT_EQ(execution->node_storage[1].payload.scalars[0], 31415U); - EXPECT_EQ(node.payload.tensors[0].version, 1618); - EXPECT_EQ(node.task.task_id, PTO2TaskId::make(1, (8U << 10U))); - EXPECT_EQ(node.task.packed_buffer_base, second_heap.data()); - EXPECT_EQ(node.payload.tensors[0].buffer.addr, reinterpret_cast(second_boundary.data())); - EXPECT_EQ( - execution->node_storage[1].payload.tensors[0].buffer.addr, reinterpret_cast(second_heap.data() + 16) - ); - EXPECT_EQ(node.slot.completed_subtasks.load(std::memory_order_relaxed), 0); - EXPECT_EQ(node.payload.dispatch_fanin.load(std::memory_order_relaxed), 0); - EXPECT_EQ(node.payload.dump_metadata.dump_arg_mask, uint64_t{1} << 0); - - graph_execution_mark_completed(*execution); - execution->retired_nodes.store(2, std::memory_order_release); - submission.local_execution = 0; - execution = graph_execution_localize(outer_slot); - ASSERT_NE(execution, nullptr); - ASSERT_TRUE(execution->definition_affine_reuse); - execution->materialized_tensor_patch_count = 1; - - EXPECT_EQ(graph_execution_materialize_slice(outer_slot, *execution, 2), GraphMaterializeResult::INVALID); - EXPECT_EQ(execution->materialized_tensor_patches, 1U); -} - -TEST(GraphSubmissionWire, RejectsDefinitionBeyondSubmission) { - constexpr uint64_t GRAPH_KEY_VALUE = 0x3456; - std::vector image = make_test_submission(GRAPH_KEY_VALUE, 0x1000, 17, 0x2000, 4096); - auto &submission = *reinterpret_cast(image.data()); - auto *definition = reinterpret_cast(image.data() + submission.definition_offset); - definition->total_bytes = submission.total_bytes - submission.definition_offset + 1; - - EXPECT_EQ(graph_submission_definition(submission), nullptr); -} - -TEST(GraphSubmissionWire, RequiresExactAvailableSize) { - constexpr uint64_t GRAPH_KEY_VALUE = 0x4567; - std::vector image = make_test_submission(GRAPH_KEY_VALUE, 0x1000, 17, 0x2000, 4096); - const auto &submission = *reinterpret_cast(image.data()); - - EXPECT_TRUE(graph_submission_wire_size_valid(submission, image.size())); - EXPECT_FALSE(graph_submission_wire_size_valid(submission, image.size() - 1)); - EXPECT_FALSE(graph_submission_wire_size_valid(submission, image.size() + 1)); -} - -TEST(GraphSubmissionActivationGate, ActivatesExactlyOnceUnderContention) { - constexpr int ITERATIONS = 1000; - for (int iteration = 0; iteration < ITERATIONS; ++iteration) { - GraphSubmission submission{}; - std::atomic activations{0}; - std::thread prepared([&] { - if (graph_submission_signal(submission, 0x1)) activations.fetch_add(1, std::memory_order_relaxed); - }); - std::thread ready([&] { - if (graph_submission_signal(submission, 0x2)) activations.fetch_add(1, std::memory_order_relaxed); - }); - prepared.join(); - ready.join(); - EXPECT_EQ(submission.activation_gate, 0x3U); - EXPECT_EQ(activations.load(std::memory_order_relaxed), 1); - } -} - -TEST(GraphSubmissionActivationGate, RetriesDoNotReactivate) { - GraphSubmission submission{}; - - EXPECT_FALSE(graph_submission_signal(submission, 0x1)); - EXPECT_TRUE(graph_submission_signal(submission, 0x2)); - EXPECT_FALSE(graph_submission_signal(submission, 0x1)); - EXPECT_FALSE(graph_submission_signal(submission, 0x2)); -} - -TEST(GraphExecutionErrors, ReadyQueueOverflowHasTriageText) { - EXPECT_STREQ(error_name(PTO2_ERROR_READY_QUEUE_OVERFLOW), "READY_QUEUE_OVERFLOW"); - EXPECT_STRNE(error_desc(PTO2_ERROR_READY_QUEUE_OVERFLOW), ""); - EXPECT_STRNE(error_hint(PTO2_ERROR_READY_QUEUE_OVERFLOW), ""); -} - -TEST(GraphExecutionErrors, GraphReadyQueueOverflowIsReported) { - PTO2SharedMemoryHeader header{}; - PTO2SchedulerState scheduler{}; - scheduler.sm_header = &header; - PTO2ReadyQueueSlot queue_slots[2]{}; - queue_slots[0].sequence.store(0, std::memory_order_relaxed); - queue_slots[1].sequence.store(1, std::memory_order_relaxed); - scheduler.graph_ready_queue.slots = queue_slots; - scheduler.graph_ready_queue.capacity = 2; - scheduler.graph_ready_queue.mask = 1; - scheduler.graph_ready_queue.enqueue_pos.store(0, std::memory_order_relaxed); - scheduler.graph_ready_queue.dequeue_pos.store(0, std::memory_order_relaxed); - PTO2TaskSlotState graph_slots[3]{}; - for (PTO2TaskSlotState &slot : graph_slots) { - slot.task_kind = TaskKind::GRAPH; - } - - scheduler.push_ready_routed(&graph_slots[0]); - scheduler.push_ready_routed(&graph_slots[1]); - scheduler.push_ready_routed(&graph_slots[2]); - - EXPECT_EQ(header.sched_error_code.load(std::memory_order_acquire), PTO2_ERROR_READY_QUEUE_OVERFLOW); -} - -TEST(GraphExecutionErrors, GraphPrepareQueueOverflowIsReported) { - PTO2SharedMemoryHeader header{}; - PTO2SchedulerState scheduler{}; - scheduler.sm_header = &header; - PTO2ReadyQueueSlot queue_slots[2]{}; - queue_slots[0].sequence.store(0, std::memory_order_relaxed); - queue_slots[1].sequence.store(1, std::memory_order_relaxed); - scheduler.graph_prepare_queue.slots = queue_slots; - scheduler.graph_prepare_queue.capacity = 2; - scheduler.graph_prepare_queue.mask = 1; - scheduler.graph_prepare_queue.enqueue_pos.store(0, std::memory_order_relaxed); - scheduler.graph_prepare_queue.dequeue_pos.store(0, std::memory_order_relaxed); - PTO2TaskSlotState graph_slots[3]{}; - - EXPECT_TRUE(scheduler.push_graph_prepare(&graph_slots[0], 10, 3)); - EXPECT_TRUE(scheduler.push_graph_prepare(&graph_slots[1], 11, 3)); - EXPECT_FALSE(scheduler.push_graph_prepare(&graph_slots[2], 12, 3)); - - EXPECT_EQ(header.sched_error_code.load(std::memory_order_acquire), PTO2_ERROR_READY_QUEUE_OVERFLOW); - EXPECT_EQ(header.sched_error_thread.load(std::memory_order_acquire), 3); - EXPECT_EQ(header.sched_error_bitmap.load(std::memory_order_acquire), 1U << 3); -} - -TEST(GraphExecutionErrors, InvalidNodeCompletionIsReported) { - PTO2SchedulerState scheduler{}; - PTO2TaskSlotState slot{}; - slot.task_kind = TaskKind::GRAPH_NODE; - - const PTO2SchedulerState::TaskCompletionOutcome outcome = scheduler.complete_task(slot); - - EXPECT_EQ(outcome.error_code, PTO2_ERROR_INVALID_ARGS); - EXPECT_EQ(outcome.stream_tasks_completed, 0); -} - -TEST(GraphExecutionProgress, InternalNodeResolutionIsNotAHostCompletion) { - PTO2SchedulerState scheduler{}; - GraphDefinition definition{}; - GraphNodeStorage node{}; - GraphExecution execution{}; - execution.definition = &definition; - execution.nodes = &node; - execution.node_storage = &node; - execution.node_count = 1; - execution.remaining_nodes.store(1, std::memory_order_relaxed); - execution.state.store(GraphExecutionState::ACTIVE, std::memory_order_relaxed); - node.slot.task_kind = TaskKind::GRAPH_NODE; - node.slot.graph_context = &execution; - node.slot.graph_node_index = 0; - - AsyncWaitList wait_list{}; - wait_list.entries[0].slot_state = &node.slot; - wait_list.entries[0].task_token = PTO2TaskId::make(0, 1); - wait_list.entries[0].normal_done = true; - wait_list.count = 1; - - const AsyncPollResult result = wait_list.poll_and_complete(nullptr, &scheduler); - - EXPECT_EQ(result.error_code, PTO2_ERROR_NONE); - EXPECT_EQ(result.resolved, 1); - EXPECT_EQ(result.completed, 0); -} diff --git a/tests/ut/cpp/a2a3/test_graph_cache.cpp b/tests/ut/cpp/common/test_hbg_graph_cache.cpp similarity index 99% rename from tests/ut/cpp/a2a3/test_graph_cache.cpp rename to tests/ut/cpp/common/test_hbg_graph_cache.cpp index 529f2c4bae..86b55c6ef4 100644 --- a/tests/ut/cpp/a2a3/test_graph_cache.cpp +++ b/tests/ut/cpp/common/test_hbg_graph_cache.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "graph_cache.h"