From e08e1b3876676dfff5483e1a368b0857a682f850 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 23 Jun 2026 12:08:09 -0700 Subject: [PATCH 1/3] Scalable Topics: typed C++ SDK public API (pulsar::st) Header-only public API for the scalable-topics SDK under a new pulsar::st namespace (PIP-460/468/483): client, producers, the three consumer modes, transactions, schemas (reflect-cpp JSON/Avro and protobuf), and the Expected/Future result types, plus examples under examples/st. API definition only -- no lib/st implementation or C API yet. The new API requires C++20; the rest of the client stays C++17. Signed-off-by: Matteo Merli --- examples/CMakeLists.txt | 30 ++ examples/st/README.md | 40 ++ examples/st/SampleStCheckpointConsumer.cc | 69 +++ examples/st/SampleStJsonSchema.cc | 85 ++++ examples/st/SampleStProducer.cc | 91 ++++ examples/st/SampleStQueueConsumer.cc | 68 +++ examples/st/SampleStStreamConsumer.cc | 63 +++ include/pulsar/st/AvroSchema.h | 76 +++ include/pulsar/st/Checkpoint.h | 121 +++++ include/pulsar/st/CheckpointConsumer.h | 318 ++++++++++++ include/pulsar/st/Client.h | 360 +++++++++++++ include/pulsar/st/Consumer.h | 77 +++ include/pulsar/st/Error.h | 103 ++++ include/pulsar/st/Expected.h | 305 +++++++++++ include/pulsar/st/Future.h | 205 ++++++++ include/pulsar/st/JsonSchema.h | 76 +++ include/pulsar/st/Message.h | 237 +++++++++ include/pulsar/st/MessageId.h | 109 ++++ include/pulsar/st/Policies.h | 155 ++++++ include/pulsar/st/Producer.h | 478 ++++++++++++++++++ include/pulsar/st/ProtobufNativeSchema.h | 71 +++ include/pulsar/st/QueueConsumer.h | 372 ++++++++++++++ include/pulsar/st/Schema.h | 275 ++++++++++ include/pulsar/st/StreamConsumer.h | 416 +++++++++++++++ include/pulsar/st/Transaction.h | 149 ++++++ .../pulsar/st/detail/CheckpointConsumerCore.h | 65 +++ include/pulsar/st/detail/ClientCore.h | 73 +++ include/pulsar/st/detail/Cxx20.h | 26 + include/pulsar/st/detail/MessageCore.h | 82 +++ include/pulsar/st/detail/ProducerCore.h | 65 +++ include/pulsar/st/detail/QueueConsumerCore.h | 68 +++ include/pulsar/st/detail/SharedState.h | 96 ++++ include/pulsar/st/detail/StreamConsumerCore.h | 69 +++ vcpkg.json | 4 + 34 files changed, 4897 insertions(+) create mode 100644 examples/st/README.md create mode 100644 examples/st/SampleStCheckpointConsumer.cc create mode 100644 examples/st/SampleStJsonSchema.cc create mode 100644 examples/st/SampleStProducer.cc create mode 100644 examples/st/SampleStQueueConsumer.cc create mode 100644 examples/st/SampleStStreamConsumer.cc create mode 100644 include/pulsar/st/AvroSchema.h create mode 100644 include/pulsar/st/Checkpoint.h create mode 100644 include/pulsar/st/CheckpointConsumer.h create mode 100644 include/pulsar/st/Client.h create mode 100644 include/pulsar/st/Consumer.h create mode 100644 include/pulsar/st/Error.h create mode 100644 include/pulsar/st/Expected.h create mode 100644 include/pulsar/st/Future.h create mode 100644 include/pulsar/st/JsonSchema.h create mode 100644 include/pulsar/st/Message.h create mode 100644 include/pulsar/st/MessageId.h create mode 100644 include/pulsar/st/Policies.h create mode 100644 include/pulsar/st/Producer.h create mode 100644 include/pulsar/st/ProtobufNativeSchema.h create mode 100644 include/pulsar/st/QueueConsumer.h create mode 100644 include/pulsar/st/Schema.h create mode 100644 include/pulsar/st/StreamConsumer.h create mode 100644 include/pulsar/st/Transaction.h create mode 100644 include/pulsar/st/detail/CheckpointConsumerCore.h create mode 100644 include/pulsar/st/detail/ClientCore.h create mode 100644 include/pulsar/st/detail/Cxx20.h create mode 100644 include/pulsar/st/detail/MessageCore.h create mode 100644 include/pulsar/st/detail/ProducerCore.h create mode 100644 include/pulsar/st/detail/QueueConsumerCore.h create mode 100644 include/pulsar/st/detail/SharedState.h create mode 100644 include/pulsar/st/detail/StreamConsumerCore.h diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 6ffb4078..81796c09 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -105,3 +105,33 @@ target_link_libraries(SampleReaderCApi ${CLIENT_LIBS} pulsarShar target_link_libraries(SampleKeyValueSchemaConsumer ${CLIENT_LIBS} pulsarShared) target_link_libraries(SampleKeyValueSchemaProducer ${CLIENT_LIBS} pulsarShared) target_link_libraries(SampleCustomLoggerCApi ${CLIENT_LIBS} pulsarShared) + +# --- Scalable topics (pulsar::st) examples --------------------------------- +# These use the new typed scalable-topics API under include/pulsar/st. Its +# implementation (lib/st) does not exist yet, so the examples are COMPILED here +# for header/API verification but are NOT linked into executables (there are no +# symbols to link against). Building this OBJECT library on every build keeps the +# examples from bit-rotting while the API is reviewed. +# +# TODO(scalable-topics): once lib/st lands, replace this with one +# add_executable + target_link_libraries(... pulsarShared) per file, exactly like +# the samples above. +set(SAMPLE_ST_SOURCES + st/SampleStProducer.cc + st/SampleStStreamConsumer.cc + st/SampleStQueueConsumer.cc + st/SampleStCheckpointConsumer.cc + st/SampleStJsonSchema.cc +) +# reflect-cpp powers jsonSchema() (reflection-based JSON SerDe + schema) and is +# a required dependency of the scalable-topics API. +find_package(reflectcpp CONFIG REQUIRED) + +add_library(StExamples OBJECT ${SAMPLE_ST_SOURCES}) +# The scalable-topics (pulsar::st) API targets C++20; the rest of the client stays +# C++17. Set the standard per-target so only this code requires C++20. +set_target_properties(StExamples PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED ON) +# PRIVATE link gives the object sources pulsarShared's and reflect-cpp's include +# directories; an OBJECT library is not itself linked, so the missing lib/st +# symbols are fine. +target_link_libraries(StExamples PRIVATE ${CLIENT_LIBS} pulsarShared reflectcpp::reflectcpp) diff --git a/examples/st/README.md b/examples/st/README.md new file mode 100644 index 00000000..f095f09d --- /dev/null +++ b/examples/st/README.md @@ -0,0 +1,40 @@ +# Scalable Topics (`pulsar::st`) — API preview examples + +These examples exercise the new typed scalable-topics C++ API under +[`include/pulsar/st/`](../../include/pulsar/st). They illustrate the proposed +surface and exist to gather community feedback. + +> **Status: API definition only.** The implementation (`lib/st/`) does not exist +> yet, so these examples **compile but do not yet link**. They are wired into the +> CMake build as a compile-only `OBJECT` library (`StExamples` in +> [`examples/CMakeLists.txt`](../CMakeLists.txt)) — header-verified on every build, +> but not linked. Once `lib/st` lands they become normal `add_executable` targets. + +The `pulsar::st` API requires **C++20** (the rest of the client stays C++17). +Syntax-check an example against the headers (no linking): + +```sh +clang++ -std=c++20 -I ../../include -Wall -fsyntax-only SampleStProducer.cc +``` + +| File | Shows | +|---|---| +| `SampleStProducer.cc` | blocking + asynchronous publishing, transactions | +| `SampleStStreamConsumer.cc` | ordered (per-key) delivery, cumulative ack | +| `SampleStQueueConsumer.cc` | parallel delivery, individual ack + nack, dead-letter | +| `SampleStCheckpointConsumer.cc`| externally held position via `Checkpoint` | +| `SampleStJsonSchema.cc` | a struct as JSON with zero boilerplate (`jsonSchema()`, reflect-cpp) | + +## API at a glance + +- **Typed builders** off one `PulsarClient`: `newProducer` / `newStreamConsumer` / + `newQueueConsumer` / `newCheckpointConsumer`, each taking a `Schema`. +- **Synchronous calls return `Expected`** (a stand-in for `std::expected`, + which is C++23): check it, or call `.value()` to throw `ClientException`. + `Expected` is `[[nodiscard]]`, so a failure cannot be silently dropped. +- **Asynchronous calls return `Future`**: `addListener(...)` to react on + completion without blocking, `get()` to block, or `co_await` it. +- **Schemas**: primitives are built in; structured types use `jsonSchema()` / + `avroSchema()` (reflect-cpp derives the SerDe **and** the declared schema from + the struct — no boilerplate), `protobufNativeSchema()`, or a custom + `Schema(serde)`. reflect-cpp is a required dependency of `pulsar::st`. diff --git a/examples/st/SampleStCheckpointConsumer.cc b/examples/st/SampleStCheckpointConsumer.cc new file mode 100644 index 00000000..7baa34c8 --- /dev/null +++ b/examples/st/SampleStCheckpointConsumer.cc @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Scalable-topics CheckpointConsumer: the application owns the position. Read, +// snapshot a Checkpoint, persist it externally, and later resume from it. + +#include + +#include +#include + +using namespace pulsar::st; + +int main() { + auto clientResult = PulsarClient::builder().serviceUrl("pulsar://localhost:6650").build(); + if (!clientResult) { + std::cerr << "failed to build client: " << clientResult.error() << "\n"; + return 1; + } + PulsarClient client = std::move(clientResult).value(); + + // Restore from a previously stored checkpoint if you have one; else start at + // the earliest message. (Checkpoint::fromByteArray(savedBytes) to resume.) + auto consumerResult = client.newCheckpointConsumer(Schema{}) + .topic("topic://public/default/orders") + .startPosition(Checkpoint::earliest()) + .create(); // NOTE: create(), not subscribe() + if (!consumerResult) { + std::cerr << "failed to create consumer: " << consumerResult.error() << "\n"; + return 1; + } + CheckpointConsumer consumer = std::move(consumerResult).value(); + + for (int i = 0; i < 5; i++) { + auto msg = consumer.receive(std::chrono::seconds(5)); + if (!msg) { + if (msg.error().result == ResultTimeout) break; + std::cerr << "receive failed: " << msg.error() << "\n"; + break; + } + std::cout << "read: " << msg->value() << "\n"; + } + + // Atomic position snapshot across all segments. Store the bytes yourself + // (Flink/Spark state backend, a file, etc.) — there is no broker-side cursor. + Checkpoint checkpoint = consumer.checkpoint(); + std::string persisted = checkpoint.toByteArray(); + std::cout << "checkpoint is " << persisted.size() << " bytes\n"; + + (void)consumer.close(); + (void)client.close(); + return 0; +} diff --git a/examples/st/SampleStJsonSchema.cc b/examples/st/SampleStJsonSchema.cc new file mode 100644 index 00000000..d647042d --- /dev/null +++ b/examples/st/SampleStJsonSchema.cc @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Passing a struct as JSON: `jsonSchema()` derives both the SerDe and the +// declared schema from the struct's fields (via reflect-cpp) — NO macros, NO base +// class, NO schema string, NO serializer. Nested structs and containers included. +// `avroSchema()` is identical for Avro. + +#include +#include + +#include +#include +#include + +// Plain value types — that is the entire schema "declaration". +struct Address { + std::string street; + std::string city; +}; +struct Order { + std::string orderId; + int quantity; + double unitPrice; + Address shipTo; // nested struct — handled automatically + std::vector tags; // container — handled automatically +}; + +using namespace pulsar::st; + +int main() { + auto clientResult = PulsarClient::builder().serviceUrl("pulsar://localhost:6650").build(); + if (!clientResult) { + std::cerr << clientResult.error() << "\n"; + return 1; + } + PulsarClient client = std::move(clientResult).value(); + + auto producerResult = + client.newProducer(jsonSchema()).topic("topic://public/default/orders").create(); + if (!producerResult) { + std::cerr << producerResult.error() << "\n"; + return 1; + } + Producer producer = std::move(producerResult).value(); + + Order order{"ord-1", 3, 9.99, {"1 Main St", "Springfield"}, {"priority", "gift"}}; + if (auto sent = producer.send(order); sent) { + std::cout << "sent " << *sent << "\n"; + } + + auto consumerResult = client.newStreamConsumer(jsonSchema()) + .topic("topic://public/default/orders") + .subscriptionName("orders-sub") + .subscribe(); + if (consumerResult) { + StreamConsumer consumer = std::move(consumerResult).value(); + if (auto msg = consumer.receive(std::chrono::seconds(5))) { + Order received = msg->value(); // decoded straight back into the struct + std::cout << received.orderId << " -> " << received.shipTo.city << "\n"; + consumer.acknowledgeCumulative(msg->id()); + } + (void)consumer.close(); + } + + (void)producer.close(); + (void)client.close(); + return 0; +} diff --git a/examples/st/SampleStProducer.cc b/examples/st/SampleStProducer.cc new file mode 100644 index 00000000..2252c3dd --- /dev/null +++ b/examples/st/SampleStProducer.cc @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Scalable-topics producer: blocking and asynchronous publishing. + +#include + +#include + +using namespace pulsar::st; + +int main() { + // One client per application; keep it for the whole lifetime. + auto clientResult = PulsarClient::builder().serviceUrl("pulsar://localhost:6650").build(); + if (!clientResult) { + std::cerr << "failed to build client: " << clientResult.error() << "\n"; + return 1; + } + PulsarClient client = std::move(clientResult).value(); + + auto producerResult = client.newProducer(Schema{}) + .topic("topic://public/default/orders") + .sendTimeout(std::chrono::seconds(30)) + .create(); + if (!producerResult) { + std::cerr << "failed to create producer: " << producerResult.error() << "\n"; + return 1; + } + Producer producer = std::move(producerResult).value(); + + // Blocking send: returns Expected (must be checked — [[nodiscard]]). + for (int i = 0; i < 10; i++) { + auto sent = producer.newMessage() + .key("order-" + std::to_string(i % 4)) // per-key ordering + .value("payload-" + std::to_string(i)) + .property("attempt", "1") + .send(); + if (sent) { + std::cout << "sent " << *sent << "\n"; + } else { + std::cerr << "send failed: " << sent.error() << "\n"; + } + } + + // Asynchronous send: react on completion without blocking. + producer.newMessage().key("order-async").value("async-payload").sendAsync().addListener( + [](const Expected& result) { + if (result) { + std::cout << "async sent " << *result << "\n"; + } else { + std::cerr << "async send failed: " << result.error() << "\n"; + } + }); + + // Transaction: produced messages become visible atomically on commit. + if (auto txnResult = client.newTransaction()) { + Transaction txn = *txnResult; + auto a = producer.newMessage().value("tx-a").transaction(txn).send(); + auto b = producer.newMessage().value("tx-b").transaction(txn).send(); + if (a && b) { + if (auto committed = txn.commit(); !committed) { + std::cerr << "commit failed: " << committed.error() << "\n"; + } + } else { + (void)txn.abort(); + } + } + + (void)producer.flush(); // await all sends issued before this call + if (auto closed = producer.close(); !closed) { + std::cerr << "close failed: " << closed.error() << "\n"; + } + (void)client.close(); + return 0; +} diff --git a/examples/st/SampleStQueueConsumer.cc b/examples/st/SampleStQueueConsumer.cc new file mode 100644 index 00000000..f8171440 --- /dev/null +++ b/examples/st/SampleStQueueConsumer.cc @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Scalable-topics QueueConsumer: parallel consumption, individual ack + nack, +// with a dead-letter policy. + +#include + +#include + +using namespace pulsar::st; + +int main() { + auto clientResult = PulsarClient::builder().serviceUrl("pulsar://localhost:6650").build(); + if (!clientResult) { + std::cerr << "failed to build client: " << clientResult.error() << "\n"; + return 1; + } + PulsarClient client = std::move(clientResult).value(); + + auto consumerResult = client.newQueueConsumer(Schema{}) + .topic("topic://public/default/orders") + .subscriptionName("shared-sub") + .deadLetterPolicy({.maxRedeliverCount = 5}) // DLQ after 5 redeliveries + .subscribe(); + if (!consumerResult) { + std::cerr << "failed to subscribe: " << consumerResult.error() << "\n"; + return 1; + } + QueueConsumer consumer = std::move(consumerResult).value(); + + // No ordering guarantee; ack each message individually, or nack to redeliver. + for (;;) { + auto msg = consumer.receive(std::chrono::seconds(5)); + if (!msg) { + if (msg.error().result == ResultTimeout) break; + std::cerr << "receive failed: " << msg.error() << "\n"; + break; + } + + const bool processed = !msg->value().empty(); + if (processed) { + consumer.acknowledge(msg->id()); // fire-and-forget; never blocks or errors + } else { + consumer.negativeAcknowledge(msg->id()); // schedule redelivery + } + } + + (void)consumer.close(); + (void)client.close(); + return 0; +} diff --git a/examples/st/SampleStStreamConsumer.cc b/examples/st/SampleStStreamConsumer.cc new file mode 100644 index 00000000..57d9f355 --- /dev/null +++ b/examples/st/SampleStStreamConsumer.cc @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Scalable-topics StreamConsumer: ordered (per-key) delivery with cumulative ack. + +#include + +#include + +using namespace pulsar::st; + +int main() { + auto clientResult = PulsarClient::builder().serviceUrl("pulsar://localhost:6650").build(); + if (!clientResult) { + std::cerr << "failed to build client: " << clientResult.error() << "\n"; + return 1; + } + PulsarClient client = std::move(clientResult).value(); + + auto consumerResult = client.newStreamConsumer(Schema{}) + .topic("topic://public/default/orders") + .subscriptionName("ordered-sub") + .subscriptionInitialPosition(SubscriptionInitialPosition::Earliest) + .subscribe(); + if (!consumerResult) { + std::cerr << "failed to subscribe: " << consumerResult.error() << "\n"; + return 1; + } + StreamConsumer consumer = std::move(consumerResult).value(); + + // Ordered delivery; a single cumulative ack advances every segment to this + // message's position (there is no individual ack in this mode). + for (int i = 0; i < 10; i++) { + auto msg = consumer.receive(std::chrono::seconds(10)); + if (!msg) { + if (msg.error().result == ResultTimeout) continue; + std::cerr << "receive failed: " << msg.error() << "\n"; + break; + } + std::cout << "key=" << msg->key().value_or("") << " value=" << msg->value() << "\n"; + consumer.acknowledgeCumulative(msg->id()); // fire-and-forget; never blocks or errors + } + + (void)consumer.close(); + (void)client.close(); + return 0; +} diff --git a/include/pulsar/st/AvroSchema.h b/include/pulsar/st/AvroSchema.h new file mode 100644 index 00000000..a0ac13eb --- /dev/null +++ b/include/pulsar/st/AvroSchema.h @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include + +#include +#include + +#include +#include + +// avroSchema() is the Avro counterpart of jsonSchema(): reflect-cpp derives +// the SerDe and the Avro schema from T's fields — no per-type serializer. The +// reflect-cpp Avro backend is assumed always present (a required dependency). +// +// NOTE: the `rfl::` calls live here (not in a lib/st .cc) because the SerDe is a +// template instantiated on the user's `T` — that instantiation must happen in the +// including TU. reflect-cpp is therefore confined to this opt-in schema header, +// not the core API headers. + +namespace pulsar::st { + +/// @cond INTERNAL +/// Internal: the reflect-cpp-backed Avro SerDe used by avroSchema(). Not part +/// of the public API. +namespace detail { +template +struct AvroSerDe { + SchemaInfo info() const { return SchemaInfo(SchemaType::AVRO, "AVRO", rfl::avro::to_schema()); } + std::string encode(const T& value) const { return rfl::avro::write(value); } + T decode(const char* data, std::size_t size) const { + return rfl::avro::read(std::string(data, size)).value(); + } +}; +} // namespace detail +/// @endcond + +/** + * @brief Creates an Avro schema for `T`, with no boilerplate. + * + * The Avro counterpart of jsonSchema(): reflect-cpp derives both the SerDe and the + * Avro schema directly from the struct's fields, with no per-type serializer. + * + * @code + * auto producer = client.newProducer(avroSchema()).topic(t).create(); + * @endcode + * + * @tparam T the struct type to serialize as Avro; its fields must be reflectable + * by reflect-cpp. + * @return a `Schema` whose `encode`/`decode` use Avro. + * @throws std::runtime_error (from reflect-cpp) at decode time if the input bytes + * are not a valid Avro encoding for `T`. + */ +template +Schema avroSchema() { + return Schema(detail::AvroSerDe{}); +} + +} // namespace pulsar::st diff --git a/include/pulsar/st/Checkpoint.h b/include/pulsar/st/Checkpoint.h new file mode 100644 index 00000000..d3a32e34 --- /dev/null +++ b/include/pulsar/st/Checkpoint.h @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include + +#include +#include + +namespace pulsar::st { + +class CheckpointImpl; +class CheckpointFactory; + +/** + * @brief An opaque, serializable position vector marking a consistent point across + * all segments of a scalable topic. + * + * A `Checkpoint` is the only position type accepted by a `CheckpointConsumer`: + * unlike `MessageId`, which identifies a single message within one segment, a + * `Checkpoint` captures the read position of *every* segment at once, so it can + * express a position that spans the whole topic. + * + * The intended workflow is store-and-restore, owned entirely by the application: + * - `CheckpointConsumer::checkpoint()` captures an atomic snapshot of the current + * per-segment read positions; + * - the application serializes the snapshot with `toByteArray()` and persists the + * bytes in its own state backend; + * - on restart it rebuilds the `Checkpoint` with `fromByteArray()` and resumes + * from it via `CheckpointConsumerBuilder::startPosition(...)`. + * + * The value is opaque: it carries no observable ledger/entry/segment structure and + * is not comparable or ordered. Timestamp-based positioning is performed + * out-of-band through the `scalable-topics seek` admin operation, not through this + * type. + */ +class PULSAR_PUBLIC Checkpoint { + public: + /** + * @brief Construct an empty/invalid checkpoint. + * + * The result is falsy under `operator bool` and must not be passed as a start + * position; use `earliest()` or `latest()`, or a value restored from + * `fromByteArray()`, instead. + */ + Checkpoint(); + + /** + * @brief Well-known sentinel positioned before the earliest available message + * of every segment. + * + * Use as a start position to replay a scalable topic from the very beginning. + * + * @return Reference to the shared earliest-position sentinel. + */ + static const Checkpoint& earliest(); + + /** + * @brief Well-known sentinel positioned after the latest published message of + * every segment. + * + * Use as a start position to consume only messages published after the + * consumer is created. This is the default start position of + * `CheckpointConsumerBuilder`. + * + * @return Reference to the shared latest-position sentinel. + */ + static const Checkpoint& latest(); + + /** + * @brief Serialize this checkpoint to a portable binary form for external + * storage. + * + * The returned bytes are an opaque blob suitable for persisting in any state + * backend; restore them later with `fromByteArray()`. + * + * @return Byte string encoding the cross-segment position. + */ + std::string toByteArray() const; + + /** + * @brief Restore a `Checkpoint` previously produced by `toByteArray()`. + * + * @param data Bytes returned by an earlier `toByteArray()` call. + * @return The reconstructed `Checkpoint`. + */ + static Checkpoint fromByteArray(const std::string& data); + + /** + * @brief Test whether this checkpoint holds a valid position. + * + * @return `true` for a sentinel or a value restored from `fromByteArray()`; + * `false` for a default-constructed (empty) checkpoint. + */ + explicit operator bool() const { return static_cast(impl_); } + + private: + friend class CheckpointFactory; + explicit Checkpoint(std::shared_ptr impl); + + std::shared_ptr impl_; +}; + +} // namespace pulsar::st diff --git a/include/pulsar/st/CheckpointConsumer.h b/include/pulsar/st/CheckpointConsumer.h new file mode 100644 index 00000000..e5bb80d7 --- /dev/null +++ b/include/pulsar/st/CheckpointConsumer.h @@ -0,0 +1,318 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace pulsar::st { + + +/** + * @brief Configuration accumulated by `CheckpointConsumerBuilder`. + * + * Populated through the builder's fluent setters and consumed by + * `CheckpointConsumerBuilder::create()` / `createAsync()`; applications normally do + * not construct this directly. + */ +struct CheckpointConsumerConfig { + std::string topic; ///< Scalable topic to read. REQUIRED; no default. + Checkpoint startPosition = Checkpoint::latest(); ///< Position to start from. Default `Checkpoint::latest()`. + std::optional consumerGroup; ///< Consumer group to join. Unset (default) => ungrouped, reads every segment. + std::optional consumerName; ///< Human-readable consumer name. Unset (default) => auto-generated. + Properties properties; ///< Free-form key/value metadata attached to the consumer. Default empty. + SchemaInfo schema; ///< Schema descriptor; filled in from `Schema` by the builder. +}; + + + + +template +class CheckpointConsumerBuilder; + +/** + * @brief Unmanaged consumer with an externally held position — reader semantics, + * no broker-managed cursor and no acknowledgment (spec §7.3). + * + * A `CheckpointConsumer` reads from every segment of a scalable topic like a + * reader: the broker keeps no durable cursor for it and there is no acknowledgment. + * The application owns the position. It captures a `Checkpoint` with + * `checkpoint()`, persists it (see `Checkpoint::toByteArray()`), and on restart + * resumes from it through `CheckpointConsumerBuilder::startPosition(...)`. + * + * Instances are created with `CheckpointConsumerBuilder::create()` / + * `createAsync()` (note: not `subscribe()`, since there is no subscription). A + * default-constructed `CheckpointConsumer` is empty and falsy under `operator + * bool`. + * + * @tparam T Message payload type; decoded according to the configured `Schema`. + */ +template +class CheckpointConsumer { + public: + /** @brief Construct an empty, unusable consumer (falsy under `operator bool`). */ + CheckpointConsumer() = default; + + /** + * @brief Block until the next message is available and return it. + * + * Waits indefinitely for the next message at the current read position. + * + * @return `Expected>` holding the decoded message, or an `Error` if + * the receive fails (e.g. the consumer is closed or disconnected, or the + * payload cannot be decoded). + */ + Expected> receive() { return toTyped(core_.receiveAsync().get()); } + + /** + * @brief Block for at most `timeout` waiting for the next message. + * + * @param timeout Maximum time to wait (`std::chrono::milliseconds`). + * @return `Expected>` holding the decoded message, or an `Error`; a + * timeout surfaces as `Error{ResultTimeout}`. May also fail on + * close/disconnect or a decode error. + */ + Expected> receive(std::chrono::milliseconds timeout) { + return toTyped(core_.receiveAsync(timeout.count()).get()); + } + + /** + * @brief Asynchronously receive the next message. + * + * Non-blocking counterpart of `receive()`; the returned future completes when a + * message is available or the receive fails. + * + * @return `Future>` resolving to the decoded message, or an `Error`. + */ + Future> receiveAsync() { + Schema schema = schema_; + return core_.receiveAsync().thenApply( + [schema](const detail::MessageCore& core) { return Message(core, schema); }); + } + + /** + * @brief Block to receive up to `maxMessages`, returning early on `timeout`. + * + * Accumulates whatever messages are available, returning as soon as + * `maxMessages` is reached or `timeout` elapses (whichever comes first). The + * returned batch may contain fewer than `maxMessages` entries. + * + * @param maxMessages Maximum number of messages to return in one call. + * @param timeout Maximum time to wait for the batch to fill + * (`std::chrono::milliseconds`). + * @return `Expected>` holding the decoded batch, or an `Error` on + * failure. + */ + Expected> receiveMulti(int maxMessages, std::chrono::milliseconds timeout) { + return toTypedBatch(core_.receiveMultiAsync(maxMessages, timeout.count()).get()); + } + + /** + * @brief Capture an atomic snapshot of the current read positions across all + * segments (spec §8). + * + * Records, as a single consistent `Checkpoint`, the per-segment positions read + * so far. The snapshot is taken locally with no broker round-trip. Persist the + * returned value (see `Checkpoint::toByteArray()`) to be able to resume from + * exactly this point later. + * + * @return A `Checkpoint` representing the current cross-segment position. + */ + Checkpoint checkpoint() const { return core_.checkpoint(); } + + /** + * @brief Close the consumer and release its resources. + * + * Blocks until the close completes. After closing, no further receives succeed. + * + * @return `Expected` holding success, or an `Error` if the close failed. + */ + Expected close() { return core_.closeAsync().get(); } + + /** + * @brief Asynchronously close the consumer. + * + * Non-blocking counterpart of `close()`. + * + * @return `Future` resolving to success, or an `Error` on failure. + */ + Future closeAsync() { return core_.closeAsync(); } + + /** + * @brief Return the topic this consumer reads from. + * + * @return Reference to the topic name. + */ + const std::string& topic() const { return core_.topic(); } + + /** + * @brief Test whether this consumer is usable. + * + * @return `true` for a consumer produced by the builder; `false` for a + * default-constructed (empty) one. + */ + explicit operator bool() const { return static_cast(core_); } + + private: + template + friend class CheckpointConsumerBuilder; + CheckpointConsumer(detail::CheckpointConsumerCore core, Schema schema) + : core_(std::move(core)), schema_(std::move(schema)) {} + + Expected> toTyped(Expected r) const { + if (r) return Message(*r, schema_); + return Expected>(r.error()); + } + Expected> toTypedBatch(Expected> r) const { + if (!r) return Expected>(r.error()); + std::vector> out; + out.reserve(r->size()); + for (auto& core : *r) out.emplace_back(core, schema_); + return Messages(std::move(out)); + } + + detail::CheckpointConsumerCore core_; + Schema schema_; +}; + +/** + * @brief Fluent builder for a `CheckpointConsumer`. + * + * Obtained from `PulsarClient`. Configure it through the chainable setters, then + * call the terminal `create()` / `createAsync()` to build the consumer. Note the + * terminal is `create()`, not `subscribe()`, because a checkpoint consumer has no + * broker-managed subscription. + * + * @tparam T Message payload type of the consumer being built. + */ +template +class CheckpointConsumerBuilder { + public: + /** + * @brief Set the scalable topic to read from. REQUIRED; no default. + * + * @param t Topic name. + * @return `*this`, for call chaining. + */ + CheckpointConsumerBuilder& topic(std::string t) { + config_.topic = std::move(t); + return *this; + } + /** + * @brief Set the position to start reading from. Default `Checkpoint::latest()`. + * + * Pass `Checkpoint::earliest()` to replay from the beginning, or a value + * restored via `Checkpoint::fromByteArray()` to resume from a persisted + * position. + * + * @param c Start position. + * @return `*this`, for call chaining. + */ + CheckpointConsumerBuilder& startPosition(Checkpoint c) { + config_.startPosition = std::move(c); + return *this; + } + /** + * @brief Join a consumer group so segments are shared across its members. + * + * Members of the same group divide the topic's segments among themselves; + * leaving this unset (the default) makes the consumer ungrouped, so it reads + * every segment on its own. + * + * @param g Consumer group name. + * @return `*this`, for call chaining. + */ + CheckpointConsumerBuilder& consumerGroup(std::string g) { + config_.consumerGroup = std::move(g); + return *this; + } + /** + * @brief Set a human-readable consumer name. Default: auto-generated when unset. + * + * @param n Consumer name. + * @return `*this`, for call chaining. + */ + CheckpointConsumerBuilder& consumerName(std::string n) { + config_.consumerName = std::move(n); + return *this; + } + /** + * @brief Attach a free-form key/value property to the consumer. + * + * May be called repeatedly to add several properties; defaults to none. + * + * @param k Property key. + * @param v Property value. + * @return `*this`, for call chaining. + */ + CheckpointConsumerBuilder& property(const std::string& k, const std::string& v) { + config_.properties[k] = v; + return *this; + } + + /** + * @brief Build the consumer, blocking until it is ready. + * + * @return `Expected>` holding the consumer, or an `Error` + * if creation failed. + */ + Expected> create() { return createAsync().get(); } + + /** + * @brief Asynchronously build the consumer. + * + * Non-blocking counterpart of `create()`. + * + * @return `Future>` resolving to the consumer, or an + * `Error` on failure. + */ + Future> createAsync() { + Schema schema = schema_; + CheckpointConsumerConfig config = config_; + config.schema = schema.info(); + return client_.createCheckpointAsync(std::move(config)) + .thenApply([schema](const detail::CheckpointConsumerCore& core) { + return CheckpointConsumer(core, schema); + }); + } + + private: + friend class PulsarClient; + CheckpointConsumerBuilder(detail::ClientCore client, Schema schema) + : client_(std::move(client)), schema_(std::move(schema)) {} + + detail::ClientCore client_; + Schema schema_; + CheckpointConsumerConfig config_; +}; + +} // namespace pulsar::st diff --git a/include/pulsar/st/Client.h b/include/pulsar/st/Client.h new file mode 100644 index 00000000..3cd82529 --- /dev/null +++ b/include/pulsar/st/Client.h @@ -0,0 +1,360 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace pulsar::st { + +class PulsarClientBuilder; + +/** + * The heavyweight, thread-safe entry point of the scalable-topics SDK. It owns the + * connection pool, IO threads, and memory buffers. An application SHOULD create a + * single instance and keep it for the whole application lifetime, sharing it + * across all producers and consumers, and close it exactly once at shutdown + * (spec §3). + * + * A `PulsarClient` is a lightweight, copyable handle to that shared state, built + * only through `PulsarClient::builder()`. + */ +class PULSAR_PUBLIC PulsarClient { + public: + /** + * Begin configuring a client. + * + * This is the only way to obtain a PulsarClient: the default constructor is + * private. Set at least PulsarClientBuilder::serviceUrl, then call + * PulsarClientBuilder::build. + * + * @return a fresh, unconfigured client builder + */ + static PulsarClientBuilder builder(); + + /** + * Start building a producer for values of type `T`. + * + * The schema governs serialization and broker-side compatibility: built-ins + * cover `Bytes` (the default), `std::string` and numeric primitives; + * structured types use `jsonSchema()`, `avroSchema()`, + * `protobufNativeSchema()`, or a custom SerDe (see Schema.h and the + * dedicated schema headers). + * + * @tparam T the value type produced; defaults to `Bytes` (raw payload). + * @param schema the schema describing how `T` is encoded; defaults to the + * built-in schema for `T`. + * @return a ProducerBuilder to further configure and create the producer + */ + template + ProducerBuilder newProducer(Schema schema = {}) { + return ProducerBuilder(core_, std::move(schema)); + } + + /** + * Start building a stream consumer: an ordered, broker-managed, + * cumulative-ack consumer (spec §7.1). + * + * @tparam T the value type consumed; defaults to `Bytes` (raw payload). + * @param schema the schema describing how `T` is decoded; defaults to the + * built-in schema for `T`. + * @return a StreamConsumerBuilder to further configure and create the consumer + */ + template + StreamConsumerBuilder newStreamConsumer(Schema schema = {}) { + return StreamConsumerBuilder(core_, std::move(schema)); + } + + /** + * Start building a queue consumer: a parallel, broker-managed, + * individual-ack consumer (spec §7.2). + * + * @tparam T the value type consumed; defaults to `Bytes` (raw payload). + * @param schema the schema describing how `T` is decoded; defaults to the + * built-in schema for `T`. + * @return a QueueConsumerBuilder to further configure and create the consumer + */ + template + QueueConsumerBuilder newQueueConsumer(Schema schema = {}) { + return QueueConsumerBuilder(core_, std::move(schema)); + } + + /** + * Start building a checkpoint consumer: an unmanaged consumer whose read + * position is held by the client rather than the broker (spec §7.3). + * + * @tparam T the value type consumed; defaults to `Bytes` (raw payload). + * @param schema the schema describing how `T` is decoded; defaults to the + * built-in schema for `T`. + * @return a CheckpointConsumerBuilder to further configure and create the consumer + */ + template + CheckpointConsumerBuilder newCheckpointConsumer(Schema schema = {}) { + return CheckpointConsumerBuilder(core_, std::move(schema)); + } + + /** + * Open a new transaction synchronously (spec §9). + * + * Blocks until the transaction has been started by the broker. The + * transaction uses the default timeout configured via + * PulsarClientBuilder::transactionPolicy. + * + * @return the new Transaction, or an Error if it could not be started + */ + Expected newTransaction() { return core_.newTransactionAsync().get(); } + + /** + * Open a new transaction asynchronously (spec §9). + * + * @return a Future that completes with the new Transaction, or an Error if it + * could not be started + */ + Future newTransactionAsync() { return core_.newTransactionAsync(); } + + /** + * Close the client gracefully. + * + * Awaits all pending operations to complete, then releases every resource the + * client owns (connections, IO threads, buffers). Blocks until the shutdown + * finishes. Call this exactly once at application shutdown. + * + * @return an empty Expected on success, or an Error if the close failed + */ + Expected close() { return core_.closeAsync().get(); } + + /** + * Close the client gracefully, asynchronously. + * + * Like close() but returns immediately with a Future that completes once all + * pending operations have drained and resources have been released. + * + * @return a Future that completes empty on success, or with an Error if the + * close failed + */ + Future closeAsync() { return core_.closeAsync(); } + + /** + * Shut the client down immediately. + * + * Drops any pending operations without waiting for them and releases + * resources at once. Prefer close() for an orderly shutdown; use this only + * when a graceful close is not possible or not desired. + */ + void shutdown() { core_.shutdown(); } + + /** + * Test whether this handle refers to a live client. + * + * @return true if the handle is backed by shared client state; false if it + * has been moved from + */ + explicit operator bool() const { return static_cast(core_); } + + private: + friend class PulsarClientBuilder; + PulsarClient() = default; + explicit PulsarClient(detail::ClientCore core) : core_(std::move(core)) {} + + detail::ClientCore core_; +}; + +/** + * Configures and builds a `PulsarClient`. `serviceUrl` is the only required + * setting; everything else has sensible defaults and is grouped into policy + * objects (spec Appendix A). With C++20 designated initializers: + * + * auto client = PulsarClient::builder() + * .serviceUrl("pulsar://localhost:6650") + * .connectionPolicy({.connectionsPerBroker = 4, .connectionTimeout = 10s}) + * .build(); + */ +class PULSAR_PUBLIC PulsarClientBuilder { + public: + /** + * REQUIRED — the Pulsar endpoint, e.g. `pulsar://localhost:6650`. + * + * This is the only required setting; build() fails if it is not set. + * + * @param url the broker or proxy service URL to connect to + * @return `*this`, for call chaining + */ + PulsarClientBuilder& serviceUrl(std::string url) { + serviceUrl_ = std::move(url); + return *this; + } + + /** + * Set the authentication provider used when connecting to the broker. + * + * Optional. When unset, the client connects without authentication. + * + * @param auth the authentication provider to use + * @return `*this`, for call chaining + */ + PulsarClientBuilder& authentication(AuthenticationPtr auth) { + authentication_ = std::move(auth); + return *this; + } + + /** + * Set the number of threads used for network IO. + * + * Optional. Defaults to 1 when unset. + * + * @param n the number of IO threads + * @return `*this`, for call chaining + */ + PulsarClientBuilder& ioThreads(int n) { + ioThreads_ = n; + return *this; + } + /** + * Set the number of threads used to run message listeners. + * + * Optional. Defaults to 1 when unset. + * + * @param n the number of message-listener threads + * @return `*this`, for call chaining + */ + PulsarClientBuilder& messageListenerThreads(int n) { + messageListenerThreads_ = n; + return *this; + } + /** + * Set the client-wide memory budget for pending (in-flight) messages. + * + * Optional. When unset, the client applies its built-in default limit. + * + * @param size the maximum memory, in bytes, for buffered messages + * @return `*this`, for call chaining + */ + PulsarClientBuilder& memoryLimit(MemorySize size) { + memoryLimit_ = size; + return *this; + } + + /** + * Set connection-pool, lookup, and request-timeout tuning. + * + * Optional. Any field left unset within the policy falls back to the client + * default for that setting. + * + * @param policy the connection policy to apply + * @return `*this`, for call chaining + */ + PulsarClientBuilder& connectionPolicy(ConnectionPolicy policy) { + connectionPolicy_ = std::move(policy); + return *this; + } + /** + * Set the reconnection backoff policy. + * + * Optional. Any field left unset within the policy falls back to the client + * default for that bound. + * + * @param policy the backoff policy to apply + * @return `*this`, for call chaining + */ + PulsarClientBuilder& backoffPolicy(BackoffPolicy policy) { + backoffPolicy_ = std::move(policy); + return *this; + } + /** + * Set the transport security (TLS) policy. + * + * Optional. When unset, TLS is disabled and connections are plaintext. + * + * @param policy the TLS policy to apply + * @return `*this`, for call chaining + */ + PulsarClientBuilder& tlsPolicy(TlsPolicy policy) { + tlsPolicy_ = std::move(policy); + return *this; + } + /** + * Set client-wide transaction settings (spec §9). + * + * Optional. Transactions are always available; this only tunes the default + * transaction timeout. When unset, the client applies its built-in default. + * + * @param policy the transaction policy to apply + * @return `*this`, for call chaining + */ + PulsarClientBuilder& transactionPolicy(TransactionPolicy policy) { + transactionPolicy_ = std::move(policy); + return *this; + } + + /** + * Set the advertised listener name to use for broker discovery. + * + * Optional. Used in multi-listener deployments to select which set of + * advertised addresses the client connects through. When unset, the broker's + * default listener is used. + * + * @param name the configured listener name + * @return `*this`, for call chaining + */ + PulsarClientBuilder& listenerName(std::string name) { + listenerName_ = std::move(name); + return *this; + } + + /** + * Build the client from the configured settings. + * + * @return the new PulsarClient on success, or an Error describing the + * configuration problem (for example, a missing or invalid + * serviceUrl()) + */ + Expected build(); + + private: + std::string serviceUrl_; + AuthenticationPtr authentication_; + std::optional ioThreads_; + std::optional messageListenerThreads_; + std::optional memoryLimit_; + ConnectionPolicy connectionPolicy_; + BackoffPolicy backoffPolicy_; + TlsPolicy tlsPolicy_; + TransactionPolicy transactionPolicy_; + std::optional listenerName_; +}; + +inline PulsarClientBuilder PulsarClient::builder() { return PulsarClientBuilder{}; } + +} // namespace pulsar::st diff --git a/include/pulsar/st/Consumer.h b/include/pulsar/st/Consumer.h new file mode 100644 index 00000000..81b430b9 --- /dev/null +++ b/include/pulsar/st/Consumer.h @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include + +#include +#include +#include + +/** + * @file + * Enumerations and policy types shared across the consumer modes (spec §7). + */ + +namespace pulsar::st { + +/** + * Where a brand-new subscription starts reading. + * + * Determines the initial position of the cursor the first time a subscription is + * created. It is ignored once the subscription exists and has a durable cursor: + * an already-established subscription always resumes from its stored position. + */ +enum class SubscriptionInitialPosition { + Earliest, ///< Start from the oldest available message on the topic. + Latest ///< Start from the newest message, skipping anything published before subscribing. +}; + +/** + * Acknowledgment tuning (mirrors the Java v5 config package). + * + * Controls how the consumer batches acknowledgments and, for a QueueConsumer, how + * long redelivery is delayed after a negative acknowledgment. Both fields are + * optional and fall back to the client default when unset. + */ +struct AckPolicy { + /** Time window over which acknowledgments are batched before being sent, in milliseconds; 0 acks immediately. Unset uses the client default. */ + std::optional groupTime; + /** Delay before a negatively-acknowledged message is redelivered, in milliseconds. QueueConsumer only. Unset uses the client default. */ + std::optional negativeAckRedeliveryDelay; +}; + +/** + * Dead-letter handling for a QueueConsumer (spec §7.2). + * + * When a message is redelivered more than #maxRedeliverCount times, it is moved + * to a dead-letter topic instead of being redelivered indefinitely. Dead-lettering + * applies to a QueueConsumer only and is disabled unless #maxRedeliverCount is set + * to a positive value. + */ +struct DeadLetterPolicy { + /** Maximum number of redeliveries before a message is routed to the dead-letter topic. Defaults to 0, which disables dead-lettering. */ + int maxRedeliverCount = 0; + /** Name of the dead-letter topic. Unset defaults to "<topic>-<subscription>-DLQ". */ + std::optional deadLetterTopic; + /** If set, creates this subscription on the dead-letter topic up front so no messages are missed before a consumer attaches. Unset creates no initial subscription. */ + std::optional initialSubscriptionName; +}; + +} // namespace pulsar::st diff --git a/include/pulsar/st/Error.h b/include/pulsar/st/Error.h new file mode 100644 index 00000000..9e90e016 --- /dev/null +++ b/include/pulsar/st/Error.h @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace pulsar::st { + +// The scalable-topics SDK reuses the existing `pulsar::Result` code taxonomy and +// the {result, message} `Error` pair rather than introducing a parallel one: the +// underlying core already emits `pulsar::Result`, so reusing it avoids a lossy +// translation layer and keeps `strResult()` / error strings consistent. C++20 +// `using enum` re-exports the result codes into `pulsar::st`, so they are usable +// unqualified here (e.g. `ResultTimeout`) — no scoped parallel enum needed. +/** Re-export of `pulsar::Error`: the `{result, message}` pair describing a failure. */ +using pulsar::Error; +/** Re-export of `pulsar::Result`: the enumeration of machine-readable result codes. */ +using pulsar::Result; +/** Re-export the `Result` enumerators into `pulsar::st` so they are usable unqualified (e.g. `ResultTimeout`). */ +using enum pulsar::Result; + +/** + * The exception type of the scalable-topics API, wrapping a {result, message} + * pair. The API is **non-throwing by default**: synchronous calls return + * `Expected` and asynchronous calls deliver `Expected` to a `Future` + * listener. `ClientException` is thrown only when you opt in — by calling + * `Expected::value()` on a result that holds an error. Code that never calls + * `value()` never throws (and the C API under `pulsar/c/st/` is fully non-throwing + * and ABI-stable). + */ +class PULSAR_PUBLIC ClientException : public std::exception { + public: + /** + * Construct from a result code and detail message. + * + * The `what()` string is composed from the code's `strResult()` name and, when + * non-empty, @p message. + * + * @param result the machine-readable result code describing the failure. + * @param message the human-readable detail message (may be empty). + */ + ClientException(Result result, std::string message) + : error_{result, std::move(message)}, + what_(error_.message.empty() ? std::string{strResult(result)} + : std::string{strResult(result)} + ": " + error_.message) {} + + /** + * Construct from an existing `{result, message}` `Error` pair. + * + * This is the constructor `Expected::value()` uses to turn a stored error + * into a thrown exception. + * + * @param error the error pair to wrap. + */ + explicit ClientException(Error error) : ClientException(error.result, std::move(error.message)) {} + + /** The machine-readable result code. */ + Result result() const noexcept { return error_.result; } + + /** The human-readable detail message (may be empty). */ + const std::string& message() const noexcept { return error_.message; } + + /** The full {result, message} pair. */ + const Error& error() const noexcept { return error_; } + + /** + * The formatted exception description, as required by `std::exception`. + * + * Combines the result code's `strResult()` name with the detail message when one + * is present. + * + * @return a null-terminated string owned by this exception. + */ + const char* what() const noexcept override { return what_.c_str(); } + + private: + Error error_; + std::string what_; +}; + +} // namespace pulsar::st diff --git a/include/pulsar/st/Expected.h b/include/pulsar/st/Expected.h new file mode 100644 index 00000000..05871493 --- /dev/null +++ b/include/pulsar/st/Expected.h @@ -0,0 +1,305 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +// A C++17 stand-in for std::expected (C++23), used as the single +// synchronous result type of the scalable-topics API. The public surface mirrors +// std::expected so call sites read identically on every toolchain; on compilers +// that ship we could later alias to std::expected without changing +// user code. The one intentional deviation: `value()` on an error throws our +// `ClientException` (not std::bad_expected_access), keeping one error channel. + +#if defined(__cpp_exceptions) || defined(_CPPUNWIND) +#define PULSAR_ST_THROW(ex) throw(ex) +#else +#define PULSAR_ST_THROW(ex) (static_cast(ex), std::abort()) +#endif + +namespace pulsar::st { + +/** Wrap an `Error` to construct an `Expected` in its error state. */ +struct Unexpected { + /** The wrapped error to seed the `Expected`'s error state. */ + Error error; +}; + +/** + * Build an `Unexpected` from an existing `Error`. + * + * Convenience factory for returning a failure from a function whose return type + * is an `Expected`, mirroring `std::unexpected`. + * + * @param error the error to wrap. + * @return an `Unexpected` carrying @p error, implicitly convertible to any `Expected`. + */ +inline Unexpected unexpected(Error error) { return Unexpected{std::move(error)}; } + +/** + * Build an `Unexpected` from a result code and an optional detail message. + * + * Convenience overload that constructs the `Error` pair in place, so call sites + * can write `return unexpected(ResultTimeout, "...")` without naming `Error`. + * + * @param result the machine-readable result code describing the failure. + * @param message an optional human-readable detail message (empty by default). + * @return an `Unexpected` carrying `Error{result, message}`. + */ +inline Unexpected unexpected(Result result, std::string message = {}) { + return Unexpected{Error{result, std::move(message)}}; +} + +/** + * Holds either a value of type `T` or an `Error`. Returned by every synchronous + * operation. `[[nodiscard]]` so a failure can't be silently dropped — the one + * weakness of value-based errors, closed at compile time. + */ +template +class [[nodiscard]] Expected { + public: + /** The contained value type. */ + using value_type = T; + /** The error type held in the failure state. */ + using error_type = Error; + + /** Construct in the value state by copying @p value. */ + Expected(const T& value) : storage_(std::in_place_index<0>, value) {} + /** Construct in the value state by moving @p value. */ + Expected(T&& value) : storage_(std::in_place_index<0>, std::move(value)) {} + /** Construct in the error state by copying @p error. */ + Expected(const Error& error) : storage_(std::in_place_index<1>, error) {} + /** Construct in the error state by moving @p error. */ + Expected(Error&& error) : storage_(std::in_place_index<1>, std::move(error)) {} + /** Construct in the error state from an `Unexpected` wrapper. */ + Expected(Unexpected u) : storage_(std::in_place_index<1>, std::move(u.error)) {} + + /** + * Whether this holds a value (as opposed to an error). + * + * @return `true` if a value is present, `false` if it holds an error. + */ + bool has_value() const noexcept { return storage_.index() == 0; } + + /** + * Whether this holds a value, for use in a boolean context. + * + * Equivalent to `has_value()`. Enables the non-throwing pattern + * `if (result) { use(*result); } else { handle(result.error()); }`. + * + * @return `true` if a value is present, `false` if it holds an error. + */ + explicit operator bool() const noexcept { return has_value(); } + + /** + * Returns the value, or throws `ClientException` if this holds an error. + * + * This mirrors `std::expected::value()`, which throws + * `std::bad_expected_access` on an error. Modern C++ has no checked + * exception specifications (dynamic specs were removed in C++17), so the thrown + * type is documented here, not declared in the signature. Under `-fno-exceptions` + * this aborts. Use `operator bool` + `operator*` for the non-throwing path. + */ + const T& value() const& { + if (!has_value()) PULSAR_ST_THROW(ClientException(std::get<1>(storage_))); + return std::get<0>(storage_); + } + T& value() & { + if (!has_value()) PULSAR_ST_THROW(ClientException(std::get<1>(storage_))); + return std::get<0>(storage_); + } + T&& value() && { + if (!has_value()) PULSAR_ST_THROW(ClientException(std::get<1>(storage_))); + return std::get<0>(std::move(storage_)); + } + + /** + * Unchecked access to the contained value. + * + * Unlike `value()`, this never throws and performs no check. Behaviour is + * undefined if this holds an error; verify with `operator bool` first. + * + * @return a reference to the contained value (lvalue or rvalue per ref-qualifier). + */ + const T& operator*() const& noexcept { return std::get<0>(storage_); } + /** @copydoc operator*() const& */ + T& operator*() & noexcept { return std::get<0>(storage_); } + /** @copydoc operator*() const& */ + T&& operator*() && noexcept { return std::get<0>(std::move(storage_)); } + + /** + * Unchecked member access to the contained value. + * + * Behaviour is undefined if this holds an error; verify with `operator bool` + * first. + * + * @return a pointer to the contained value. + */ + const T* operator->() const noexcept { return std::get_if<0>(&storage_); } + /** @copydoc operator->() const */ + T* operator->() noexcept { return std::get_if<0>(&storage_); } + + /** + * Access the contained error. + * + * @pre `!has_value()`. Behaviour is undefined if this holds a value. + * @return a reference to the contained `Error`. + */ + const Error& error() const& noexcept { return std::get<1>(storage_); } + /** @copydoc error() const& */ + Error& error() & noexcept { return std::get<1>(storage_); } + + /** + * Return the contained value, or @p fallback if this holds an error. + * + * @tparam U a type convertible to `T`. + * @param fallback the value to return when no value is present. + * @return a copy of the contained value, or `static_cast(fallback)` on error. + */ + template + T value_or(U&& fallback) const& { + return has_value() ? std::get<0>(storage_) : static_cast(std::forward(fallback)); + } + + /** + * Monadic chaining: invoke @p f on the value, or propagate the error. + * + * If this holds a value, returns `f(value)` — which must itself be an + * `Expected`. If this holds an error, the error is forwarded unchanged into a + * fresh `Expected` of @p f's return type, and @p f is not called. Mirrors + * `std::expected::and_then`. + * + * @tparam F a callable taking `const T&` and returning some `Expected`. + * @param f the continuation to invoke on the value. + * @return `f(value)` on success, or that result type's error state on failure. + */ + template + auto and_then(F&& f) const& { + using R = std::remove_cv_t>>; + return has_value() ? std::forward(f)(std::get<0>(storage_)) : R(error()); + } + + /** + * Monadic mapping: transform the value through @p f, or propagate the error. + * + * If this holds a value, returns `Expected(f(value))` where `U` is @p f's + * return type. If this holds an error, the error is forwarded unchanged and + * @p f is not called. Mirrors `std::expected::transform`. + * + * @tparam F a callable taking `const T&` and returning a plain value `U`. + * @param f the mapping function to apply to the value. + * @return `Expected` holding `f(value)` on success, or the error on failure. + */ + template + auto transform(F&& f) const& { + using U = std::remove_cv_t>>; + return has_value() ? Expected(std::forward(f)(std::get<0>(storage_))) + : Expected(error()); + } + + /** + * Monadic error recovery: invoke @p f on the error, or pass the value through. + * + * If this holds an error, returns `f(error)` — used to recover or substitute an + * alternative result. If this holds a value, `*this` is returned unchanged and + * @p f is not called. Mirrors `std::expected::or_else`. + * + * @tparam F a callable taking `const Error&` and returning an `Expected`. + * @param f the recovery function to invoke on the error. + * @return `*this` on success, or `f(error)` on failure. + */ + template + Expected or_else(F&& f) const& { + return has_value() ? *this : std::forward(f)(error()); + } + + private: + std::variant storage_; +}; + +/** + * Specialization for value-less results (`close`, `flush`, `commit`, …). + * + * Carries no value: it is either in the success state or holds an `Error`. Used as + * the synchronous result type of operations that either succeed or fail without + * producing data. + */ +template <> +class [[nodiscard]] Expected { + public: + /** The (absent) value type. */ + using value_type = void; + /** The error type held in the failure state. */ + using error_type = Error; + + /** Construct in the success state. */ + Expected() noexcept = default; // success + /** Construct in the error state by copying @p error. */ + Expected(const Error& error) : error_(error) {} + /** Construct in the error state by moving @p error. */ + Expected(Error&& error) : error_(std::move(error)) {} + /** Construct in the error state from an `Unexpected` wrapper. */ + Expected(Unexpected u) : error_(std::move(u.error)) {} + + /** + * Whether this represents success (as opposed to an error). + * + * @return `true` on success, `false` if it holds an error. + */ + bool has_value() const noexcept { return !error_.has_value(); } + + /** + * Whether this represents success, for use in a boolean context. + * + * @return `true` on success, `false` if it holds an error. + */ + explicit operator bool() const noexcept { return has_value(); } + + /** + * Throw `ClientException` if this holds an error; otherwise return normally. + * + * The value-less analogue of `Expected::value()`: it has no value to yield, + * so on success it simply returns. As with the primary template, the thrown + * type is documented rather than declared in the signature; under + * `-fno-exceptions` this aborts. + */ + void value() const { + if (!has_value()) PULSAR_ST_THROW(ClientException(*error_)); + } + + /** + * Access the contained error. + * + * @pre `!has_value()`. Behaviour is undefined on a success value. + * @return a reference to the contained `Error`. + */ + const Error& error() const& noexcept { return *error_; } + + private: + std::optional error_; +}; + +} // namespace pulsar::st diff --git a/include/pulsar/st/Future.h b/include/pulsar/st/Future.h new file mode 100644 index 00000000..000f06f9 --- /dev/null +++ b/include/pulsar/st/Future.h @@ -0,0 +1,205 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace pulsar::st { + +namespace detail { +template +class Promise; +} + +/** + * The result of an asynchronous operation, available now or later. + * + * Unlike `std::future`, this is **continuation-capable**: `addListener()` runs a + * callback when the operation completes, so you can react without blocking. The + * callback receives an `Expected` (value or error). `get()` is available when + * you do want to block. On a C++20 toolchain the future is also `co_await`-able. + * + * A Future is cheap to copy (it shares the underlying state). Listeners run on + * whichever thread completes the operation — do not block inside one. + * + * @tparam T the type of the value the operation produces (`void` for value-less + * operations, which complete with an `Expected`). + */ +template +class Future { + public: + /** Callback type accepted by `addListener`, invoked with `const Expected&`. */ + using Listener = typename detail::SharedState::Listener; + + /** + * Register a continuation to run when the operation completes. + * + * @p listener is invoked with the `Expected` result (value or error) on the + * thread that completes the operation, or synchronously on the calling thread if + * the operation has already completed. It does not block. Do not block inside + * the listener. Multiple listeners may be registered. + * + * @param listener the callback to invoke on completion. + * @return `*this`, to allow chaining. + */ + Future& addListener(Listener listener) { + state_->addListener(std::move(listener)); + return *this; + } + + /** + * Block the calling thread until the operation completes and return its result. + * + * @return the `Expected` result, holding either the value or the error. + */ + Expected get() const { return state_->get(); } + + /** + * Block until the operation completes or @p timeout elapses, whichever first. + * + * @tparam Rep the `std::chrono::duration` representation type. + * @tparam Period the `std::chrono::duration` period type. + * @param timeout the maximum time to wait for completion. + * @return the `Expected` result if it completed in time, or `std::nullopt` + * if @p timeout elapsed first. + */ + template + std::optional> get(std::chrono::duration timeout) const { + return state_->get(timeout); + } + + /** + * Whether the operation has already completed. + * + * @return `true` if the result is available (so `get()` would not block), + * `false` otherwise. + */ + bool isReady() const { return state_->isReady(); } + + /** + * Return a new Future whose value is `f` applied to this one's value on + * completion; an error propagates unchanged. Lets a typed facade map a Future + * of a (non-templated) core into a Future of its typed wrapper. + * + * @p f is applied on whichever thread completes this Future. If this Future + * completes with an error, @p f is not called and the error is forwarded to the + * returned Future. Only participates in overload resolution when `T` is not + * `void`. + * + * @tparam F a callable taking `const T&` and returning the mapped value. + * @tparam U defaults to `T`; an implementation detail of the `void` constraint. + * @param f the mapping function to apply to the value. + * @return a `Future` of `f`'s return type, completed with the mapped value on + * success or the propagated error on failure. + */ + template , int> = 0> + Future> thenApply(F f) const { + using R = std::invoke_result_t; + detail::Promise promise; + state_->addListener([promise, f = std::move(f)](const Expected& result) { + if (result) { + promise.setValue(f(*result)); + } else { + promise.setError(result.error()); + } + }); + return promise.getFuture(); + } + + /** + * Coroutine support: whether the awaiting coroutine may skip suspension. + * + * Part of the C++20 awaitable interface so a `Future` can be used as + * `Expected r = co_await someFuture;`. Not called directly. + * + * @return `true` if the result is already available, `false` otherwise. + */ + bool await_ready() const { return state_->isReady(); } + + /** + * Coroutine support: suspend the awaiting coroutine until completion. + * + * Registers a listener that resumes @p handle when the operation completes. Part + * of the C++20 awaitable interface; not called directly. + * + * @param handle the suspended coroutine to resume on completion. + */ + void await_suspend(std::coroutine_handle<> handle) { + state_->addListener([handle](const Expected&) { handle.resume(); }); + } + + /** + * Coroutine support: produce the value of a `co_await` expression. + * + * Part of the C++20 awaitable interface; not called directly. + * + * @return the `Expected` result of the awaited operation. + */ + Expected await_resume() const { return state_->get(); } + + private: + template + friend class detail::Promise; + explicit Future(std::shared_ptr> state) : state_(std::move(state)) {} + + std::shared_ptr> state_; +}; + +namespace detail { + +/** + * INTERNAL producing side of a `Future`. The SDK fulfils these to complete the + * futures it returns; applications only ever consume `Future` and never build + * a Promise, so it lives in `detail` rather than on the public surface. + */ +template +class Promise { + public: + Promise() : state_(std::make_shared>()) {} + + Future getFuture() const { return Future(state_); } + + bool complete(Expected result) const { return state_->complete(std::move(result)); } + bool setError(Error error) const { return state_->complete(Expected(std::move(error))); } + + template , int> = 0> + bool setValue(U value) const { + return state_->complete(Expected(std::move(value))); + } + + template , int> = 0> + bool setSuccess() const { + return state_->complete(Expected()); + } + + private: + std::shared_ptr> state_; +}; + +} // namespace detail +} // namespace pulsar::st diff --git a/include/pulsar/st/JsonSchema.h b/include/pulsar/st/JsonSchema.h new file mode 100644 index 00000000..03af1a22 --- /dev/null +++ b/include/pulsar/st/JsonSchema.h @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include + +#include +#include + +#include +#include + +// jsonSchema() derives BOTH the JSON SerDe and the declared schema from T's +// fields via reflect-cpp (https://github.com/getml/reflect-cpp) — no per-type +// serializer, no schema string. This is the Jackson-equivalent for the Java +// client: you pass a plain struct and it just works. reflect-cpp is a required +// dependency of the scalable-topics API (C++20). + +namespace pulsar::st { + +/// @cond INTERNAL +/// Internal: the reflect-cpp-backed JSON SerDe used by jsonSchema(). Not part +/// of the public API. +namespace detail { +template +struct JsonSerDe { + SchemaInfo info() const { return SchemaInfo(SchemaType::JSON, "JSON", rfl::json::to_schema()); } + std::string encode(const T& value) const { return rfl::json::write(value); } + T decode(const char* data, std::size_t size) const { + return rfl::json::read(std::string(data, size)).value(); + } +}; +} // namespace detail +/// @endcond + +/** + * @brief Creates a JSON schema for `T`, with no boilerplate. + * + * reflect-cpp derives both the JSON SerDe and the declared schema directly from + * the struct's fields (nested structs and containers included) — there is no + * per-type serializer or hand-written schema string. This is the equivalent of the + * Java client's Jackson-based JSON schema: pass a plain struct and it just works. + * + * @code + * struct Order { std::string id; int qty; }; // the whole "declaration" + * auto producer = client.newProducer(jsonSchema()).topic(t).create(); + * @endcode + * + * @tparam T the struct type to serialize as JSON; its fields must be reflectable + * by reflect-cpp. + * @return a `Schema` whose `encode`/`decode` use JSON. + * @throws std::runtime_error (from reflect-cpp) at decode time if the input bytes + * are not valid JSON for `T`. + */ +template +Schema jsonSchema() { + return Schema(detail::JsonSerDe{}); +} + +} // namespace pulsar::st diff --git a/include/pulsar/st/Message.h b/include/pulsar/st/Message.h new file mode 100644 index 00000000..055ba7a7 --- /dev/null +++ b/include/pulsar/st/Message.h @@ -0,0 +1,237 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace pulsar::st { + +/** + * A message received from a scalable topic, carrying a value of type `T`. + * + * The value is decoded lazily through `Schema` on each `value()` call; the raw + * bytes and all metadata are available without decoding. + * + * @tparam T the application type the payload decodes to, via `Schema`. + */ +template +class Message { + public: + /** Construct an empty message (`operator bool` is `false`). */ + Message() = default; + + /** + * Wrap a received core payload together with the schema used to decode it. + * + * Constructed by the SDK on delivery; applications obtain `Message` objects from + * a consumer rather than building them. + * + * @param core the raw received message (payload and metadata). + * @param schema the schema used to decode the payload in `value()`. + */ + Message(detail::MessageCore core, Schema schema) : core_(std::move(core)), schema_(std::move(schema)) {} + + /** + * Decode the payload through `Schema` and return the typed value. + * + * Decoding happens on every call (the result is not cached). May throw if the + * payload bytes are malformed for the schema. + * + * @return the decoded value of type `T`. + */ + T value() const { return schema_.decode(core_.data(), core_.size()); } + + /** + * Pointer to the raw, undecoded payload bytes. + * + * @return a pointer to `size()` bytes of payload, valid for this message's lifetime. + */ + const char* data() const { return core_.data(); } + + /** + * Size of the raw payload in bytes. + * + * @return the number of bytes pointed to by `data()`. + */ + std::size_t size() const { return core_.size(); } + + /** + * The message's position within the topic. + * + * @return the `MessageId` identifying this message. + */ + MessageId id() const { return core_.id(); } + + /** + * The optional partition/routing key the message was published with. + * + * @return the key, or `std::nullopt` if the message has none. + */ + std::optional key() const { + return core_.hasKey() ? std::optional(core_.key()) : std::nullopt; + } + + /** + * The application-defined string properties attached to the message. + * + * @return a reference to the properties map. + */ + const Properties& properties() const { return core_.properties(); } + + /** + * The broker-assigned publish time. + * + * @return the timestamp at which the message was published. + */ + Timestamp publishTime() const { return Timestamp(std::chrono::milliseconds(core_.publishTimeMs())); } + + /** + * The optional application-supplied event time. + * + * @return the event time, or `std::nullopt` if the producer did not set one. + */ + std::optional eventTime() const { + auto ms = core_.eventTimeMs(); + return ms != 0 ? std::optional(Timestamp(std::chrono::milliseconds(ms))) : std::nullopt; + } + + /** + * The producer-assigned sequence id of the message. + * + * @return the sequence id. + */ + int64_t sequenceId() const { return core_.sequenceId(); } + + /** + * The name of the producer that published the message, if available. + * + * @return the producer name, or `std::nullopt` if not present. + */ + std::optional producerName() const { + return core_.hasProducerName() ? std::optional(core_.producerName()) : std::nullopt; + } + + /** + * The resolved canonical topic the message was received from. + * + * @return the fully-qualified topic name. + */ + std::string topic() const { return core_.topic(); } + + /** + * How many times this message has been redelivered. + * + * @return the redelivery count (0 on first delivery). + */ + int redeliveryCount() const { return core_.redeliveryCount(); } + + /** + * The source cluster this message was replicated from, if any. + * + * @return the originating cluster name, or `std::nullopt` if the message was not + * replicated from another cluster. + */ + std::optional replicatedFrom() const { + return core_.hasReplicatedFrom() ? std::optional(core_.replicatedFrom()) : std::nullopt; + } + + /** + * Whether this is a non-empty message. + * + * @return `true` for a real received message, `false` for a default-constructed one. + */ + explicit operator bool() const { return static_cast(core_); } + + private: + detail::MessageCore core_; + Schema schema_; +}; + +/** + * An iterable batch of messages, as returned by `receiveMulti`. Carries up to the + * requested count. + * + * @tparam T the application type of each contained `Message`. + */ +template +class Messages { + public: + /** The element type of the batch. */ + using value_type = Message; + /** Const iterator over the contained messages. */ + using const_iterator = typename std::vector>::const_iterator; + + /** + * Construct a batch wrapping the given messages. + * + * @param messages the messages to hold (empty by default). + */ + explicit Messages(std::vector> messages = {}) : messages_(std::move(messages)) {} + + /** + * The number of messages in the batch. + * + * @return the element count. + */ + std::size_t size() const { return messages_.size(); } + + /** + * Whether the batch contains no messages. + * + * @return `true` if empty, `false` otherwise. + */ + bool empty() const { return messages_.empty(); } + + /** + * Access the message at index @p i. + * + * @param i a zero-based index, which must be less than `size()`. + * @return a reference to the message at that index. + */ + const Message& operator[](std::size_t i) const { return messages_[i]; } + + /** + * Iterator to the first message. + * + * @return a const iterator to the beginning of the batch. + */ + const_iterator begin() const { return messages_.begin(); } + + /** + * Iterator past the last message. + * + * @return a const iterator to the end of the batch. + */ + const_iterator end() const { return messages_.end(); } + + private: + std::vector> messages_; +}; + +} // namespace pulsar::st diff --git a/include/pulsar/st/MessageId.h b/include/pulsar/st/MessageId.h new file mode 100644 index 00000000..52abacb8 --- /dev/null +++ b/include/pulsar/st/MessageId.h @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace pulsar::st { + +class MessageIdImpl; +class MessageIdFactory; + +/** + * The identifier of a single message within a scalable topic. + * + * It is **opaque** — no ledger/entry/segment structure is exposed. Internally it + * encodes the segment the message belongs to (so a later ack can be routed to the + * right segment), but that is not part of the contract. A `MessageId` is: + * - serializable, via `toByteArray()` / `fromByteArray()`, for external storage; + * - totally ordered within a single topic, via the comparison operators. + * + * For a consistent position across *all* segments of a topic, use `Checkpoint`, + * not `MessageId` — a single id cannot express a multi-segment position. + */ +class PULSAR_PUBLIC MessageId { + public: + /** Construct an empty/invalid id (compares equal only to other empty ids). */ + MessageId(); + + /** Sentinel: the earliest (oldest) message available in the topic. */ + static const MessageId& earliest(); + + /** Sentinel: the latest (most recently published) message in the topic. */ + static const MessageId& latest(); + + /** Serialize to a portable binary form for external storage. */ + std::string toByteArray() const; + + /** Restore a `MessageId` previously produced by `toByteArray()`. */ + static MessageId fromByteArray(const std::string& data); + + // Totally ordered within a topic; `<=>` and `==` synthesize <, <=, >, >=, !=. + /** + * Three-way comparison establishing a total order within a single topic. + * + * Synthesizes `<`, `<=`, `>`, and `>=`. Ordering of ids from different topics is + * unspecified. + * + * @param other the id to compare against. + * @return the relative ordering of `*this` and @p other. + */ + std::strong_ordering operator<=>(const MessageId& other) const; + + /** + * Equality comparison; also synthesizes `!=`. + * + * @param other the id to compare against. + * @return `true` if the two ids denote the same position. + */ + bool operator==(const MessageId& other) const; + + /** + * Whether this is a valid (non-empty) id. + * + * @return `true` for a real id, `false` for a default-constructed empty one. + */ + explicit operator bool() const { return static_cast(impl_); } + + private: + friend class MessageIdFactory; + explicit MessageId(std::shared_ptr impl); + + /** + * Write a human-readable representation of @p messageId to @p s. + * + * Intended for logging and debugging; the format is not a stable contract and + * must not be parsed (use `toByteArray()` for serialization). + * + * @param s the output stream to write to. + * @param messageId the id to format. + * @return the stream @p s, to allow chaining. + */ + friend PULSAR_PUBLIC std::ostream& operator<<(std::ostream& s, const MessageId& messageId); + + std::shared_ptr impl_; +}; + +} // namespace pulsar::st diff --git a/include/pulsar/st/Policies.h b/include/pulsar/st/Policies.h new file mode 100644 index 00000000..5ffb44ba --- /dev/null +++ b/include/pulsar/st/Policies.h @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include + +#include +#include +#include +#include + +/** + * @file + * Grouped client configuration ("policies"), mirroring the Java v5 `config` + * package (spec Appendix A). These are plain aggregates, so C++20 designated + * initializers read cleanly at the call site: + * + * client.builder() + * .serviceUrl("pulsar://localhost:6650") + * .connectionPolicy({.connectionsPerBroker = 4, .connectionTimeout = 10s}) + * .tlsPolicy({.enabled = true, .trustCertsFilePath = "/etc/ca.pem"}) + * .build(); + * + * Durations are stored as std::chrono types (not raw counts); any coarser unit + * converts implicitly (e.g. `30s` into a milliseconds field). + */ + +namespace pulsar::st { + +/** + * A quantity of bytes (mirrors Java `MemorySize`). + * + * A plain aggregate wrapping a byte count, used wherever the configuration takes + * a memory budget (e.g. PulsarClientBuilder::memoryLimit). Prefer the `of*` + * factory helpers over writing a raw byte literal, as they make the unit explicit + * at the call site. + */ +struct MemorySize { + /** The size, expressed in bytes. Defaults to 0. */ + std::uint64_t bytes = 0; + + /** + * Construct a MemorySize from a count of bytes. + * + * @param b the size in bytes + * @return a MemorySize of @p b bytes + */ + static constexpr MemorySize ofBytes(std::uint64_t b) { return {b}; } + + /** + * Construct a MemorySize from a count of kibibytes (1 KiB = 1024 bytes). + * + * @param k the size in kibibytes + * @return a MemorySize of @p k * 1024 bytes + */ + static constexpr MemorySize ofKiB(std::uint64_t k) { return {k * 1024}; } + + /** + * Construct a MemorySize from a count of mebibytes (1 MiB = 1024 * 1024 bytes). + * + * @param m the size in mebibytes + * @return a MemorySize of @p m * 1024 * 1024 bytes + */ + static constexpr MemorySize ofMiB(std::uint64_t m) { return {m * 1024 * 1024}; } +}; + +/** + * Connection-pool, lookup, and request-timeout tuning. + * + * Every field is optional: when left unset the client applies its built-in + * default for that setting. Populate only the fields you wish to override, using + * C++20 designated initializers. + */ +struct ConnectionPolicy { + /** Number of physical connections opened to each broker. Unset uses the client default. */ + std::optional connectionsPerBroker; + /** Maximum time to wait for a TCP/TLS connection to be established, in milliseconds. Unset uses the client default. */ + std::optional connectionTimeout; + /** Maximum time to wait for a broker request (e.g. produce/consume control ops) to complete, in milliseconds. Unset uses the client default. */ + std::optional operationTimeout; + /** Interval between keep-alive pings sent on an idle connection, in seconds. Unset uses the client default. */ + std::optional keepAliveInterval; + /** Maximum number of concurrent topic-lookup requests in flight. Unset uses the client default. */ + std::optional maxLookupRequests; + /** Maximum number of lookup redirects to follow before failing a lookup. Unset uses the client default. */ + std::optional maxLookupRedirects; + /** Time an idle pooled connection may stay open before being closed, in milliseconds. Unset uses the client default. */ + std::optional maxConnectionIdleTime; +}; + +/** + * Reconnection backoff (mirrors Java `BackoffPolicy`). + * + * Controls the exponential delay applied between automatic reconnection attempts + * after a connection is lost. Both fields are optional; when unset the client + * applies its built-in default for that bound. + */ +struct BackoffPolicy { + /** Delay before the first reconnection attempt, in milliseconds. Unset uses the client default. */ + std::optional initialBackoff; + /** Upper bound on the backoff delay as it grows across retries, in milliseconds. Unset uses the client default. */ + std::optional maxBackoff; +}; + +/** + * Transport security (mirrors Java `TlsPolicy`). + * + * Configures TLS for connections to the broker, including the trust store and an + * optional client certificate for mutual TLS (mTLS). TLS is off unless #enabled + * is set to true. + */ +struct TlsPolicy { + /** Whether TLS is used for broker connections. Defaults to false (plaintext). */ + bool enabled = false; + /** Path to the PEM file of trusted CA certificates used to verify the broker. Unset uses the system trust store. */ + std::optional trustCertsFilePath; + /** Path to the client certificate PEM file, for mutual TLS. Unset disables client-certificate authentication. */ + std::optional certificateFilePath; + /** Path to the client private key PEM file, for mutual TLS. Unset disables client-certificate authentication. */ + std::optional privateKeyFilePath; + /** Whether to accept the broker's certificate without validating it against the trust store. Defaults to false (validation enforced). */ + bool allowInsecureConnection = false; + /** Whether to verify that the broker's certificate hostname matches the endpoint. Defaults to true. */ + bool validateHostname = true; +}; + +/** + * Client-wide transaction settings (spec §9). + * + * Transactions are always available; this policy only tunes the default + * transaction timeout applied to transactions opened by the client. The field is + * optional and the client supplies a built-in default when it is unset. + */ +struct TransactionPolicy { + /** Default lifetime of a transaction before it is automatically aborted, in milliseconds. Unset uses the client default. */ + std::optional timeout; +}; + +} // namespace pulsar::st diff --git a/include/pulsar/st/Producer.h b/include/pulsar/st/Producer.h new file mode 100644 index 00000000..0fe4fc10 --- /dev/null +++ b/include/pulsar/st/Producer.h @@ -0,0 +1,478 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace pulsar::st { + +/** + * How a producer claims write access to a topic (spec Appendix A / §4.1). + * + * The access mode is fixed at creation time via `ProducerBuilder::accessMode` + * and controls how the broker arbitrates between multiple producers on the same + * topic. + */ +enum class ProducerAccessMode { + /** Multiple producers may publish to the topic concurrently. The default. */ + Shared, + /** Only one producer may be active at a time; another producer requesting + * `Exclusive` access is rejected while one is already attached. */ + Exclusive, + /** Like `Exclusive`, but a new producer fences out (evicts) any currently + * attached producer rather than being rejected. */ + ExclusiveWithFencing, + /** Like `Exclusive`, but instead of failing when another producer holds the + * topic, this producer waits in line and becomes active once that one + * detaches. */ + WaitForExclusive +}; + +/** + * Producer configuration accumulated by `ProducerBuilder`. + * + * Applications do not populate this directly; the builder fills it in from its + * fluent setters and hands the completed config to the client when creating a + * producer. Field semantics, units and defaults mirror the corresponding builder + * setters. + */ +struct ProducerConfig { + /** Fully-qualified topic to produce to. REQUIRED; no default. */ + std::string topic; + /** Optional producer name. Unset (the default) lets the broker assign one. */ + std::optional producerName; + /** Write-access arbitration mode. Defaults to `ProducerAccessMode::Shared`. */ + ProducerAccessMode accessMode = ProducerAccessMode::Shared; + /** Per-message send timeout in milliseconds. Unset uses the SDK default. */ + std::optional sendTimeoutMs; + /** When true (the default), block the caller while the send queue is full + * instead of failing fast with `ResultProducerQueueIsFull`. */ + bool blockIfQueueFull = true; + /** Sequence id to assign to the first published message. Unset starts from + * the broker-tracked value (0 for a fresh producer). */ + std::optional initialSequenceId; + /** Arbitrary user metadata attached to the producer. Empty by default. */ + Properties properties; + /** Schema descriptor sent to the broker for compatibility checking. Filled in + * by the builder from `Schema::info()`. */ + SchemaInfo schema; +}; + +/** + * A byte-oriented outgoing message assembled by `MessageBuilder`. + * + * This is the encoded, schema-agnostic form of a message: the typed value has + * already been serialized to `payload` bytes. The builder fills these fields from + * its fluent setters and hands the result to the producer core for publishing. + */ +struct OutgoingMessage { + /** Encoded message payload (the value serialized through `Schema`). */ + std::string payload; + /** Whether a routing/ordering key is set. `false` (the default) means no key. */ + bool hasKey = false; + /** Partition/ordering key; meaningful only when `hasKey` is true. */ + std::string key; + /** Per-message user metadata. Empty by default. */ + Properties properties; + int64_t eventTimeMs = 0; ///< Application event time, epoch ms; 0 = unset. + int64_t sequenceId = -1; ///< Explicit sequence id; -1 = auto-assign. + int64_t deliverAtMs = 0; ///< Absolute delivery time, epoch ms; 0 = deliver immediately. + /** Target clusters for geo-replication; empty applies the topic's default. */ + std::vector replicationClusters; + std::optional transaction; ///< Enlisting transaction; unset = non-transactional. +}; + +template +class Producer; + +/** + * Fluent builder for a single message, obtained from `Producer::newMessage()`. + * + * Each setter mutates the in-progress message and returns `*this`, so calls can + * be chained. The typed value is encoded through `Schema` on `value()`; the + * terminal `send()` / `sendAsync()` hand the encoded message to the producer. A + * builder describes a single message and is consumed by its terminal call (the + * message is moved out), so it should not be reused afterwards. + */ +template +class MessageBuilder { + public: + /** + * Set the message key, used for per-key ordering and key-affinity routing. + * + * @param k the key; taken by value and moved into the message. + * @return `*this`, for chaining. + */ + MessageBuilder& key(std::string k) { + message_.hasKey = true; + message_.key = std::move(k); + return *this; + } + /** + * Set the message value, encoding it to bytes through this producer's + * `Schema`. + * + * @param v the typed value to publish. + * @return `*this`, for chaining. + */ + MessageBuilder& value(const T& v) { + message_.payload = schema_.encode(v); + return *this; + } + /** + * Add or overwrite a single user property on the message. + * + * @param k property key. + * @param v property value. + * @return `*this`, for chaining. + */ + MessageBuilder& property(const std::string& k, const std::string& v) { + message_.properties[k] = v; + return *this; + } + /** + * Replace all user properties on the message. + * + * @param p the full property map; taken by value and moved in. + * @return `*this`, for chaining. + */ + MessageBuilder& properties(Properties p) { + message_.properties = std::move(p); + return *this; + } + /** + * Set the application-defined event time of the message. + * + * @param t the event time as a wall-clock `Timestamp`; stored as epoch + * milliseconds. Unset by default (event time absent). + * @return `*this`, for chaining. + */ + MessageBuilder& eventTime(Timestamp t) { + message_.eventTimeMs = toEpochMs(t); + return *this; + } + /** + * Set an explicit sequence id for this message, overriding auto-assignment. + * + * @param s the sequence id. By default (-1) the producer assigns one + * automatically. + * @return `*this`, for chaining. + */ + MessageBuilder& sequenceId(int64_t s) { + message_.sequenceId = s; + return *this; + } + /** + * Request delayed delivery: deliver the message after `delay` has elapsed from + * now (spec §4 delayed delivery). + * + * @param delay delay relative to the current time, in milliseconds. Computed + * into an absolute delivery time. Mutually exclusive with + * `deliverAt`; the last of the two called wins. + * @return `*this`, for chaining. + */ + MessageBuilder& deliverAfter(std::chrono::milliseconds delay) { + message_.deliverAtMs = toEpochMs(std::chrono::system_clock::now()) + delay.count(); + return *this; + } + /** + * Request delayed delivery at a specific wall-clock time (spec §4 delayed + * delivery). + * + * @param t absolute delivery time; stored as epoch milliseconds. A time in the + * past delivers immediately. Mutually exclusive with `deliverAfter`; + * the last of the two called wins. + * @return `*this`, for chaining. + */ + MessageBuilder& deliverAt(Timestamp t) { + message_.deliverAtMs = toEpochMs(t); + return *this; + } + /** + * Enlist this publish in a transaction so it becomes visible only on commit + * (spec §9). + * + * @param txn the open transaction to enlist the publish in. + * @return `*this`, for chaining. + */ + MessageBuilder& transaction(const Transaction& txn) { + message_.transaction = txn; + return *this; + } + + /** + * Publish the message and block until the broker acknowledges it. + * + * @return the assigned `MessageId` on success, or the `Error` on failure. Call + * `.value()` on the result to throw `ClientException` instead. + */ + Expected send() { return sendAsync().get(); } + /** + * Publish the message asynchronously without blocking. + * + * @return a `Future` that completes with the assigned id on success + * or the failure. The future may be ignored for fire-and-forget sends. + */ + Future sendAsync() { return core_.sendAsync(std::move(message_)); } + + private: + friend class Producer; + MessageBuilder(detail::ProducerCore core, Schema schema) + : core_(std::move(core)), schema_(std::move(schema)) {} + + static int64_t toEpochMs(Timestamp t) { + return std::chrono::duration_cast(t.time_since_epoch()).count(); + } + + detail::ProducerCore core_; + Schema schema_; + OutgoingMessage message_; +}; + +/** + * A typed producer for a single scalable topic. + * + * Publishes values of type `T`, encoding each through its `Schema`. A producer + * is a lightweight, copyable handle over shared state; a default-constructed + * producer is empty (see `operator bool`) and only a producer obtained from + * `ProducerBuilder::create()` / `createAsync()` is live. All publish methods + * are thread-safe. + */ +template +class Producer { + public: + /** Construct an empty producer; `operator bool` is false until assigned a + * live producer from `ProducerBuilder`. */ + Producer() = default; + + /** + * Begin building a single message with per-message options (key, properties, + * event time, delayed delivery, transaction, ...). + * + * @return a fresh `MessageBuilder` bound to this producer. + */ + MessageBuilder newMessage() { return MessageBuilder(core_, schema_); } + + /** + * Publish a value and block until the broker acknowledges it. Convenience for + * `newMessage().value(value).send()`. + * + * @param value the value to publish. + * @return the assigned `MessageId` on success, or the `Error` on failure. Call + * `.value()` on the result to throw `ClientException` instead. + */ + Expected send(const T& value) { return newMessage().value(value).send(); } + /** + * Publish a value asynchronously without blocking. Convenience for + * `newMessage().value(value).sendAsync()`. + * + * @param value the value to publish. + * @return a `Future` that completes with the assigned id or the + * failure. May be ignored for fire-and-forget sends. + */ + Future sendAsync(const T& value) { return newMessage().value(value).sendAsync(); } + + /** @return the topic this producer publishes to. */ + const std::string& topic() const { return core_.topic(); } + /** @return the producer's name (broker-assigned when none was configured). */ + const std::string& name() const { return core_.name(); } + /** @return the sequence id of the most recently published message, or -1 if + * none has been published yet. */ + int64_t lastSequenceId() const { return core_.lastSequenceId(); } + + /** + * Block until all sends issued before this call have completed. Takes a + * snapshot of in-flight sends at the time of the call; sends issued afterwards + * are not awaited. + * + * @return success, or the first `Error` among the awaited sends. Call + * `.value()` to throw `ClientException` instead. + */ + Expected flush() { return core_.flushAsync().get(); } + /** + * Asynchronously await all sends issued before this call (a snapshot of + * in-flight sends). + * + * @return a `Future` that completes once those sends finish. + */ + Future flushAsync() { return core_.flushAsync(); } + /** + * Block until pending sends complete, then release the producer. Idempotent: + * closing an already-closed or empty producer succeeds. + * + * @return success, or the `Error` if closing failed. Call `.value()` to throw + * `ClientException` instead. + */ + Expected close() { return core_.closeAsync().get(); } + /** + * Asynchronously complete pending sends and release the producer. Idempotent. + * + * @return a `Future` that completes once the producer is closed. + */ + Future closeAsync() { return core_.closeAsync(); } + + /** @return true if this is a live producer handle; false if empty (default + * constructed or moved-from). */ + explicit operator bool() const { return static_cast(core_); } + + private: + template + friend class ProducerBuilder; + Producer(detail::ProducerCore core, Schema schema) : core_(std::move(core)), schema_(std::move(schema)) {} + + detail::ProducerCore core_; + Schema schema_; +}; + +/** + * Builder for a `Producer`, obtained from `PulsarClient::newProducer`. + * + * Each setter returns `*this` for chaining. `topic` is the only required setting; + * the terminal `create()` / `createAsync()` produce the `Producer`. + */ +template +class ProducerBuilder { + public: + /** + * Set the topic to produce to. REQUIRED; there is no default. + * + * @param t the fully-qualified topic name; taken by value and moved in. + * @return `*this`, for chaining. + */ + ProducerBuilder& topic(std::string t) { + config_.topic = std::move(t); + return *this; + } + /** + * Set an explicit producer name. + * + * @param n the producer name; taken by value and moved in. Optional; when + * unset (the default) the broker assigns a name. + * @return `*this`, for chaining. + */ + ProducerBuilder& producerName(std::string n) { + config_.producerName = std::move(n); + return *this; + } + /** + * Set the write-access arbitration mode (spec Appendix A / §4.1). + * + * @param m the access mode. Defaults to `ProducerAccessMode::Shared`. + * @return `*this`, for chaining. + */ + ProducerBuilder& accessMode(ProducerAccessMode m) { + config_.accessMode = m; + return *this; + } + /** + * Set the per-message send timeout: how long a send may stay unacknowledged + * before failing. + * + * @param d the timeout, in milliseconds. Optional; when unset the SDK default + * applies. + * @return `*this`, for chaining. + */ + ProducerBuilder& sendTimeout(std::chrono::milliseconds d) { + config_.sendTimeoutMs = d.count(); + return *this; + } + + /** + * Control behavior when the producer's send queue is full. + * + * @param b when true (the DEFAULT), block the caller until the queue drains; + * when false, fail fast with `ResultProducerQueueIsFull`. + * @return `*this`, for chaining. + */ + ProducerBuilder& blockIfQueueFull(bool b) { + config_.blockIfQueueFull = b; + return *this; + } + /** + * Set the sequence id assigned to the first published message. + * + * @param s the initial sequence id. Optional; when unset the producer starts + * from the broker-tracked value (0 for a fresh producer). + * @return `*this`, for chaining. + */ + ProducerBuilder& initialSequenceId(int64_t s) { + config_.initialSequenceId = s; + return *this; + } + /** + * Add or overwrite a single user property on the producer. + * + * @param k property key. + * @param v property value. + * @return `*this`, for chaining. + */ + ProducerBuilder& property(const std::string& k, const std::string& v) { + config_.properties[k] = v; + return *this; + } + + /** + * Create the producer, blocking until it is ready. + * + * @return the live `Producer` on success, or the `Error` on failure. Call + * `.value()` on the result to throw `ClientException` instead. + */ + Expected> create() { return createAsync().get(); } + /** + * Create the producer asynchronously without blocking. + * + * @return a `Future>` that completes with the live producer on + * success or the failure. + */ + Future> createAsync() { + Schema schema = schema_; + ProducerConfig config = config_; + config.schema = schema.info(); + return client_.createProducerAsync(std::move(config)).thenApply([schema](const detail::ProducerCore& core) { + return Producer(core, schema); + }); + } + + private: + friend class PulsarClient; + ProducerBuilder(detail::ClientCore client, Schema schema) + : client_(std::move(client)), schema_(std::move(schema)) {} + + detail::ClientCore client_; + Schema schema_; + ProducerConfig config_; +}; + +} // namespace pulsar::st diff --git a/include/pulsar/st/ProtobufNativeSchema.h b/include/pulsar/st/ProtobufNativeSchema.h new file mode 100644 index 00000000..348ad051 --- /dev/null +++ b/include/pulsar/st/ProtobufNativeSchema.h @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace pulsar::st { + +/// @cond INTERNAL +/// Internal: the protobuf-backed SerDe used by protobufNativeSchema(). Not part +/// of the public API. +namespace detail { +template +struct ProtobufNativeSerDe { + static_assert(std::is_base_of_v, + "protobufNativeSchema requires T to be a generated protobuf Message"); + SchemaInfo info() const { return pulsar::createProtobufNativeSchema(T::descriptor()); } + std::string encode(const T& value) const { return value.SerializeAsString(); } + T decode(const char* data, std::size_t size) const { + T message; + message.ParseFromArray(data, static_cast(size)); + return message; + } +}; +} // namespace detail +/// @endcond + +/** + * @brief Creates a schema for a generated protobuf message type `T`. + * + * Unlike JSON/Avro, the SerDe is **fully automatic**: protobuf itself provides the + * serialization, and the broker schema is derived from the message descriptor, so + * no per-type mapping or reflection library is needed. + * + * @code + * auto producer = client.newProducer(protobufNativeSchema()).topic(t).create(); + * @endcode + * + * @tparam T the message type; must derive from `google::protobuf::Message` (a + * generated protobuf class). This is enforced at compile time. + * @return a `Schema` whose `encode`/`decode` use protobuf's native wire format. + */ +template +Schema protobufNativeSchema() { + return Schema(detail::ProtobufNativeSerDe{}); +} + +} // namespace pulsar::st diff --git a/include/pulsar/st/QueueConsumer.h b/include/pulsar/st/QueueConsumer.h new file mode 100644 index 00000000..c82f2256 --- /dev/null +++ b/include/pulsar/st/QueueConsumer.h @@ -0,0 +1,372 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace pulsar::st { + + +/** + * Plain-old-data configuration accumulated by `QueueConsumerBuilder`. + * + * Each field mirrors a builder setter. Prefer building through + * `QueueConsumerBuilder` rather than populating this struct directly; the + * builder enforces the invariants (notably that exactly one of topic vs. namespace + * mode is selected and that `subscriptionName` is set). + */ +struct QueueConsumerConfig { + /// Selects namespace mode over single-topic mode. When `false` (the default), + /// `topic` is used; when `true`, `namespaceName` (and `propertyFilters`) apply. + bool useNamespace = false; + /// Fully-qualified topic name. Used only when `useNamespace == false`. Mutually + /// exclusive with `namespaceName`. + std::string topic; // when !useNamespace + /// Namespace name (`tenant/namespace`). Used only when `useNamespace == true`. + /// Subscribes to all scalable topics in the namespace with live membership. + std::string namespaceName; // when useNamespace + /// Namespace mode only: AND filters matched against topic properties to select + /// which topics in the namespace are included. Empty means no filtering (all + /// topics). Ignored in single-topic mode. + Properties propertyFilters; // namespace mode: AND filters over topic properties + /// REQUIRED. Subscription name shared by all consumers of this subscription. + std::string subscriptionName; // REQUIRED + /// Where the subscription starts when it is first created. Default + /// `SubscriptionInitialPosition::Latest` (skip the backlog). Has no effect once + /// the subscription already exists. + SubscriptionInitialPosition initialPosition = SubscriptionInitialPosition::Latest; + /// Optional consumer name (useful for diagnostics and metrics). Default unset, in + /// which case the broker assigns one. + std::optional consumerName; + /// Acknowledgment tuning (e.g. the ack-grouping/batching window and negative-ack + /// redelivery delay). Default-constructed `AckPolicy` when unset. + AckPolicy ackPolicy; + /// Optional dead-letter policy: route messages to a dead-letter topic after + /// repeated redelivery. Default unset (no dead-lettering). + std::optional deadLetterPolicy; + /// Arbitrary client-side consumer properties (reported in topic stats). Default empty. + Properties properties; + /// Schema descriptor for the value type `T`. Populated automatically by the builder + /// from the `Schema` it was constructed with. + SchemaInfo schema; +}; + + + + +template +class QueueConsumerBuilder; + +/** + * Parallel, broker-managed consumer with **individual ack + nack + dead-letter**. + * + * This is the analog of a classic Shared subscription, attached to all segments of + * a scalable topic (spec §7.2): work is distributed across all consumers on the + * subscription with no ordering or key affinity. Each message is acknowledged + * individually with `acknowledge`, can be negatively acknowledged with + * `negativeAcknowledge` to schedule redelivery, and can be routed to a dead-letter + * topic after repeated redelivery (see `QueueConsumerBuilder::deadLetterPolicy`). + * For ordered, cumulative-ack consumption use `StreamConsumer` instead. + * + * Obtain an instance from `QueueConsumerBuilder`. The default-constructed + * consumer is an empty handle (`operator bool` is `false`) until assigned one. + * + * @tparam T the decoded message value type, determined by the `Schema` used to + * build this consumer. + */ +template +class QueueConsumer { + public: + /** Construct an empty, non-live handle. `operator bool` returns `false` until a + * subscribed consumer is move-assigned into it. */ + QueueConsumer() = default; + + /** + * Block until the next message arrives and return it. + * + * Returns `Expected` because a receive can fail *without* yielding a message — + * the consumer was closed, the connection dropped, or the payload failed to + * decode. On such failures the result holds an `Error` instead of a message; + * call `.value()` on the result if you would rather throw a `ClientException`. + * + * @return the next `Message`, or an `Error` describing why no message could + * be delivered. + */ + Expected> receive() { return toTyped(core_.receiveAsync().get()); } + /** + * Block for the next message, but for no longer than `timeout`. + * + * @param timeout maximum time to wait for a message. + * @return the next `Message`; if no message arrives within `timeout`, an + * `Error{ResultTimeout}`; or another `Error` on close/disconnect/decode + * failure. + */ + Expected> receive(std::chrono::milliseconds timeout) { + return toTyped(core_.receiveAsync(timeout.count()).get()); + } + /** + * Request the next message without blocking. + * + * @return a `Future>` completed with the message when one is + * available, or completed with an `Error` (via the future's `Expected` + * result) on close/disconnect/decode failure. + */ + Future> receiveAsync() { + Schema schema = schema_; + return core_.receiveAsync().thenApply( + [schema](const detail::MessageCore& core) { return Message(core, schema); }); + } + + /** + * Individually acknowledge one message. + * + * Fire-and-forget: it does not block and does not report an error. Acks are + * buffered and delivered best-effort; a lost ack simply causes redelivery. + * + * @param id the id of the message to acknowledge. + */ + void acknowledge(const MessageId& id) { core_.acknowledge(id); } + /** + * Transactional individual acknowledge: enlist the ack of `id` in `txn`. + * + * The acknowledgment becomes effective only when `txn` commits; its outcome (and + * any error) surfaces at `Transaction::commit()`, not here. + * + * @param id the id of the message to acknowledge. + * @param txn the transaction the acknowledgment is enlisted in. + */ + void acknowledge(const MessageId& id, const Transaction& txn) { core_.acknowledge(id, txn); } + + /** + * Negatively acknowledge a message, scheduling it for redelivery. + * + * Fire-and-forget `void`: it does not block and does not report an error. The + * redelivery delay is governed by the configured `AckPolicy`. After enough + * redeliveries the message may be sent to the dead-letter topic if a + * `DeadLetterPolicy` was configured. + * + * @param id the id of the message to redeliver. + */ + void negativeAcknowledge(const MessageId& id) { core_.negativeAcknowledge(id); } + + /** + * Close the consumer, releasing its broker-side resources. Blocking. + * + * @return an empty `Expected` on success, or an `Error` if the close + * failed. Call `.value()` to throw instead. + */ + Expected close() { return core_.closeAsync().get(); } + /** + * Close the consumer without blocking. + * + * @return a `Future` completed when the close finishes (or with an `Error` + * on failure). + */ + Future closeAsync() { return core_.closeAsync(); } + + /** @return the topic this consumer is subscribed to. In namespace mode this is the + * namespace-derived subscription target. */ + const std::string& topic() const { return core_.topic(); } + /** @return the subscription name. */ + const std::string& subscription() const { return core_.subscription(); } + /** @return the consumer name (broker-assigned if none was set on the builder). */ + const std::string& consumerName() const { return core_.consumerName(); } + + /** @return `true` if this is a live, subscribed consumer; `false` if it is an empty + * (default-constructed or closed/moved-from) handle. */ + explicit operator bool() const { return static_cast(core_); } + + private: + template + friend class QueueConsumerBuilder; + QueueConsumer(detail::QueueConsumerCore core, Schema schema) + : core_(std::move(core)), schema_(std::move(schema)) {} + + Expected> toTyped(Expected r) const { + if (r) return Message(*r, schema_); + return Expected>(r.error()); + } + + detail::QueueConsumerCore core_; + Schema schema_; +}; + +/** + * Fluent builder for a `QueueConsumer`. + * + * Obtain one from `PulsarClient`. Set **exactly one** of `topic()` or + * `inNamespace()` to choose the subscription target, set the REQUIRED + * `subscriptionName()`, then call `subscribe()` / `subscribeAsync()` to create the + * consumer. All setters return `*this` for chaining. + * + * @tparam T the decoded message value type, fixed by the `Schema` the builder + * was created with. + */ +template +class QueueConsumerBuilder { + public: + /** + * Subscribe to a single scalable topic. Mutually exclusive with `inNamespace()`; + * set exactly one. Calling this clears any namespace selection. + * + * @param t the fully-qualified topic name. + * @return `*this` for chaining. + */ + QueueConsumerBuilder& topic(std::string t) { + config_.useNamespace = false; + config_.topic = std::move(t); + return *this; + } + /** + * Subscribe to all scalable topics in a namespace with live membership — topics + * created or removed later are joined/dropped automatically (spec §7.2). + * Mutually exclusive with `topic()`; set exactly one. + * + * Named `inNamespace` because `namespace` is a C++ keyword. + * + * @param ns the namespace (`tenant/namespace`) to subscribe across. + * @param propertyFilters optional AND filters matched against topic properties; + * only topics matching all entries are included. Default empty (no + * filtering — every topic in the namespace). + * @return `*this` for chaining. + */ + QueueConsumerBuilder& inNamespace(std::string ns, Properties propertyFilters = {}) { + config_.useNamespace = true; + config_.namespaceName = std::move(ns); + config_.propertyFilters = std::move(propertyFilters); + return *this; + } + /** + * REQUIRED. Set the subscription name shared by all consumers of this + * subscription. + * + * @param s the subscription name. + * @return `*this` for chaining. + */ + QueueConsumerBuilder& subscriptionName(std::string s) { + config_.subscriptionName = std::move(s); + return *this; + } + /** + * Set where the subscription starts when first created. + * + * @param p the initial position. Default `SubscriptionInitialPosition::Latest`. + * Has no effect if the subscription already exists. + * @return `*this` for chaining. + */ + QueueConsumerBuilder& subscriptionInitialPosition(SubscriptionInitialPosition p) { + config_.initialPosition = p; + return *this; + } + /** + * Set an explicit consumer name (useful for diagnostics and metrics). + * + * @param n the consumer name. Default unset, in which case the broker assigns one. + * @return `*this` for chaining. + */ + QueueConsumerBuilder& consumerName(std::string n) { + config_.consumerName = std::move(n); + return *this; + } + /** + * Tune acknowledgment behavior (e.g. the ack-grouping/batching window and the + * negative-ack redelivery delay). + * + * @param policy the ack policy. Default-constructed `AckPolicy` when unset. + * @return `*this` for chaining. + */ + QueueConsumerBuilder& ackPolicy(AckPolicy policy) { + config_.ackPolicy = std::move(policy); + return *this; + } + /** + * Route messages to a dead-letter topic after repeated redelivery (spec §7.2). + * QueueConsumer only. + * + * @param policy the dead-letter policy (max redeliveries, DLQ topic name, etc.). + * Default unset (no dead-lettering). + * @return `*this` for chaining. + */ + QueueConsumerBuilder& deadLetterPolicy(DeadLetterPolicy policy) { + config_.deadLetterPolicy = std::move(policy); + return *this; + } + /** + * Add a single client-side consumer property (reported in topic stats). Call + * repeatedly to add multiple; a repeated key overwrites the previous value. + * + * @param k property key. + * @param v property value. + * @return `*this` for chaining. + */ + QueueConsumerBuilder& property(const std::string& k, const std::string& v) { + config_.properties[k] = v; + return *this; + } + + /** + * Create the consumer and subscribe. Blocking. + * + * @return the live `QueueConsumer` on success, or an `Error` if the + * subscription failed (e.g. missing `subscriptionName`, both/neither of + * topic and namespace set, or a broker error). Call `.value()` to throw + * instead. + */ + Expected> subscribe() { return subscribeAsync().get(); } + /** + * Create the consumer and subscribe without blocking. + * + * @return a `Future>` completed with the live consumer, or with + * an `Error` on failure. + */ + Future> subscribeAsync() { + Schema schema = schema_; + QueueConsumerConfig config = config_; + config.schema = schema.info(); + return client_.subscribeQueueAsync(std::move(config)) + .thenApply([schema](const detail::QueueConsumerCore& core) { return QueueConsumer(core, schema); }); + } + + private: + friend class PulsarClient; + QueueConsumerBuilder(detail::ClientCore client, Schema schema) + : client_(std::move(client)), schema_(std::move(schema)) {} + + detail::ClientCore client_; + Schema schema_; + QueueConsumerConfig config_; +}; + +} // namespace pulsar::st diff --git a/include/pulsar/st/Schema.h b/include/pulsar/st/Schema.h new file mode 100644 index 00000000..1ad5d649 --- /dev/null +++ b/include/pulsar/st/Schema.h @@ -0,0 +1,275 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pulsar::st { + +using pulsar::SchemaInfo; +using pulsar::SchemaType; + +/** + * @brief The default value type: a raw, uninterpreted byte payload. + * + * Alias for `std::vector`. A `Schema` (the default schema) passes the + * payload through verbatim in both directions, applying no encoding or schema + * declaration to the broker beyond `SchemaType::BYTES`. + */ +using Bytes = std::vector; + +/** + * @brief Constraint identifying a *SerDe* for `T`: a type that can describe `T` to + * the broker and convert it to and from bytes. + * + * A SerDe is any copyable type providing the three const members required below. + * `Schema` is constructible from any value satisfying `SerDeFor`, + * which is how the same `T` can be carried by different encodings (JSON, Avro, + * protobuf, or a fully custom codec). + * + * The required members are: + * - `SchemaInfo info() const` — the schema description sent to the broker for + * compatibility checking. + * - `std::string encode(const T&) const` — serializes a value of `T` to bytes. + * - `T decode(const char*, std::size_t) const` — deserializes bytes back to `T`. + * + * @tparam S the candidate SerDe type. + * @tparam T the value type the SerDe handles. + */ +template +concept SerDeFor = requires(const S& serde, const T& value, const char* data, std::size_t size) { + { serde.info() } -> std::convertible_to; + { serde.encode(value) } -> std::convertible_to; + { serde.decode(data, size) } -> std::convertible_to; +}; + +/** + * `Schema` is the typed seam of the API: `Producer`, `Consumer` and + * `Message` are thin facades that only ever call `Schema::encode` / `decode` + * (and `info`, sent to the broker for compatibility). It is a lightweight, + * copyable **value** that holds a type-erased *SerDe* — so the same `T` can be + * carried by different encodings, and a producer can be handed any schema. + * + * A SerDe is any copyable type providing three const members: + * SchemaInfo info() const; // describes T to the broker + * std::string encode(const T& value) const; // T -> bytes + * T decode(const char* data, size_t) const; // bytes -> T + * + * Construct a `Schema` from a SerDe directly, or use a factory: + * - primitives: `Schema{}`, `Schema{}`, `Schema{}` (default) + * - `jsonSchema()` — (reflect-cpp; no trait) + * - `avroSchema()` — (reflect-cpp; no trait) + * - `protobufNativeSchema()` — (automatic) + * - or a custom SerDe: `Schema(mySerDe)` for full control. + * + * JSON/Avro for a user struct are derived automatically from the struct's fields + * by reflect-cpp; protobuf uses the generated message's own reflection. + */ +template +class Schema { + public: + /** @brief The value type carried by this schema. */ + using value_type = T; + + /** + * @brief Constructs the default schema for `T`. + * + * For a primitive `T` this installs the built-in codec: `Bytes` (the default), + * `std::string`, `std::int32_t`, `std::int64_t`, or `double`. Integers and + * `double` are encoded as fixed-width big-endian, matching the Pulsar wire + * format for primitive schemas. + * + * For any other (non-primitive) `T` this installs an "unset" schema: it reports + * `SchemaType::BYTES` to the broker, but its `encode` and `decode` throw + * `ClientException` on use. Supply a real schema (`jsonSchema()`, + * `avroSchema()`, `protobufNativeSchema()`, or a custom SerDe) before + * producing or consuming such a `T`. + */ + Schema(); + + /** + * @brief Constructs a schema from any SerDe value. + * + * Wraps and type-erases @p serde so that this `Schema` forwards `info`, + * `encode` and `decode` to it. Use this to plug in a custom encoding for `T`. + * + * @tparam SerDe a copyable type satisfying `SerDeFor`. + * @param serde the SerDe to adopt; taken by value and stored. + */ + template + requires(!std::is_same_v, Schema> && SerDeFor, T>) + Schema(SerDe serde) : self_(std::make_shared>>(std::move(serde))) {} + + /** + * @brief Returns the schema description sent to the broker for compatibility. + * @return the `SchemaInfo` (type, name and schema definition) describing `T`. + */ + SchemaInfo info() const { return self_->info(); } + + /** + * @brief Serializes a value to its wire bytes. + * @param value the value to encode. + * @return the encoded payload as a byte string. + * @throws ClientException if this is an unset schema (non-primitive `T` with no + * SerDe supplied). A custom SerDe may throw on its own encoding errors. + */ + std::string encode(const T& value) const { return self_->encode(value); } + + /** + * @brief Deserializes wire bytes back into a value of `T`. + * @param data pointer to the payload bytes. + * @param size number of bytes available at @p data. + * @return the decoded value. + * @throws ClientException if this is an unset schema (non-primitive `T` with no + * SerDe supplied). A SerDe may also throw on malformed or incompatible + * bytes. + */ + T decode(const char* data, std::size_t size) const { return self_->decode(data, size); } + + private: + struct Concept { + virtual ~Concept() = default; + virtual SchemaInfo info() const = 0; + virtual std::string encode(const T&) const = 0; + virtual T decode(const char*, std::size_t) const = 0; + }; + template + struct Model final : Concept { + SerDe serde; + explicit Model(SerDe s) : serde(std::move(s)) {} + SchemaInfo info() const override { return serde.info(); } + std::string encode(const T& v) const override { return serde.encode(v); } + T decode(const char* d, std::size_t n) const override { return serde.decode(d, n); } + }; + + std::shared_ptr self_; +}; + +/// @cond INTERNAL +/// Internal implementation details: built-in primitive codecs and helpers. Not +/// part of the public API. +namespace detail { + +// Pulsar encodes numeric schemas as fixed-width big-endian. +template +inline std::string encodeBigEndian(U value) { + static_assert(std::is_integral_v, "integral only"); + std::string out(sizeof(U), '\0'); + auto u = static_cast>(value); + for (std::size_t i = 0; i < sizeof(U); ++i) { + out[i] = static_cast((u >> (8 * (sizeof(U) - 1 - i))) & 0xFF); + } + return out; +} +template +inline U decodeBigEndian(const char* data, std::size_t size) { + static_assert(std::is_integral_v, "integral only"); + std::make_unsigned_t u = 0; + for (std::size_t i = 0; i < sizeof(U) && i < size; ++i) { + u = (u << 8) | static_cast(data[i]); + } + return static_cast(u); +} + +[[noreturn]] inline void throwNoSchema() { +#if defined(__cpp_exceptions) || defined(_CPPUNWIND) + throw ClientException(pulsar::ResultInvalidConfiguration, + "no schema configured for this value type — pass an explicit Schema " + "(jsonSchema/avroSchema/protobufNativeSchema, or a custom SerDe)"); +#else + std::abort(); +#endif +} + +// Built-in SerDe codecs. +struct BytesCodec { + SchemaInfo info() const { return SchemaInfo(SchemaType::BYTES, "BYTES", ""); } + std::string encode(const Bytes& v) const { return std::string(v.begin(), v.end()); } + Bytes decode(const char* d, std::size_t n) const { return Bytes(d, d + n); } +}; +struct StringCodec { + SchemaInfo info() const { return SchemaInfo(SchemaType::STRING, "String", ""); } + std::string encode(const std::string& v) const { return v; } + std::string decode(const char* d, std::size_t n) const { return std::string(d, n); } +}; +struct Int32Codec { + SchemaInfo info() const { return SchemaInfo(SchemaType::INT32, "INT32", ""); } + std::string encode(std::int32_t v) const { return encodeBigEndian(v); } + std::int32_t decode(const char* d, std::size_t n) const { return decodeBigEndian(d, n); } +}; +struct Int64Codec { + SchemaInfo info() const { return SchemaInfo(SchemaType::INT64, "INT64", ""); } + std::string encode(std::int64_t v) const { return encodeBigEndian(v); } + std::int64_t decode(const char* d, std::size_t n) const { return decodeBigEndian(d, n); } +}; +struct DoubleCodec { + SchemaInfo info() const { return SchemaInfo(SchemaType::DOUBLE, "Double", ""); } + std::string encode(double v) const { + std::uint64_t bits; + std::memcpy(&bits, &v, sizeof(bits)); + return encodeBigEndian(static_cast(bits)); + } + double decode(const char* d, std::size_t n) const { + auto bits = static_cast(decodeBigEndian(d, n)); + double v; + std::memcpy(&v, &bits, sizeof(v)); + return v; + } +}; +template +struct UnsetCodec { + SchemaInfo info() const { return SchemaInfo(SchemaType::BYTES, "BYTES", ""); } + std::string encode(const T&) const { throwNoSchema(); } + T decode(const char*, std::size_t) const { throwNoSchema(); } +}; + +} // namespace detail +/// @endcond + +template +Schema::Schema() { + if constexpr (std::is_same_v) { + self_ = std::make_shared>(detail::BytesCodec{}); + } else if constexpr (std::is_same_v) { + self_ = std::make_shared>(detail::StringCodec{}); + } else if constexpr (std::is_same_v) { + self_ = std::make_shared>(detail::Int32Codec{}); + } else if constexpr (std::is_same_v) { + self_ = std::make_shared>(detail::Int64Codec{}); + } else if constexpr (std::is_same_v) { + self_ = std::make_shared>(detail::DoubleCodec{}); + } else { + self_ = std::make_shared>>(detail::UnsetCodec{}); + } +} + +} // namespace pulsar::st diff --git a/include/pulsar/st/StreamConsumer.h b/include/pulsar/st/StreamConsumer.h new file mode 100644 index 00000000..a0f7c198 --- /dev/null +++ b/include/pulsar/st/StreamConsumer.h @@ -0,0 +1,416 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace pulsar::st { + + +/** + * Plain-old-data configuration accumulated by `StreamConsumerBuilder`. + * + * Each field mirrors a builder setter. Prefer building through + * `StreamConsumerBuilder` rather than populating this struct directly; the + * builder enforces the invariants (notably that exactly one of topic vs. + * namespace mode is selected and that `subscriptionName` is set). + */ +struct StreamConsumerConfig { + /// Selects namespace mode over single-topic mode. When `false` (the default), + /// `topic` is used; when `true`, `namespaceName` (and `propertyFilters`) apply. + bool useNamespace = false; + /// Fully-qualified topic name. Used only when `useNamespace == false`. Mutually + /// exclusive with `namespaceName`. + std::string topic; // when !useNamespace + /// Namespace name (`tenant/namespace`). Used only when `useNamespace == true`. + /// Subscribes to all scalable topics in the namespace with live membership. + std::string namespaceName; // when useNamespace + /// Namespace mode only: AND filters matched against topic properties to select + /// which topics in the namespace are included. Empty means no filtering (all + /// topics). Ignored in single-topic mode. + Properties propertyFilters; // namespace mode: AND filters over topic properties + /// REQUIRED. Subscription name shared by all consumers of this subscription. + std::string subscriptionName; // REQUIRED + /// Where the subscription starts when it is first created. Default + /// `SubscriptionInitialPosition::Latest` (skip the backlog). Has no effect once + /// the subscription already exists. + SubscriptionInitialPosition initialPosition = SubscriptionInitialPosition::Latest; + /// Optional key/value properties attached to the subscription itself (persisted + /// broker-side). Default empty. + Properties subscriptionProperties; + /// Optional consumer name (useful for diagnostics and metrics). Default unset, in + /// which case the broker assigns one. + std::optional consumerName; + /// Acknowledgment tuning (e.g. the ack-grouping/batching window). Default-constructed + /// `AckPolicy` when unset. + AckPolicy ackPolicy; + /// When set to `true`, read from the topic's compacted view (latest value per key) + /// instead of the full log. Default unset (broker default, i.e. uncompacted). + std::optional readCompacted; + /// When set to `true`, replicate the subscription's acknowledged position across + /// geo-replication clusters. Default unset (broker default, i.e. disabled). + std::optional replicateSubscriptionState; + /// Arbitrary client-side consumer properties (reported in topic stats). Default empty. + Properties properties; + /// Schema descriptor for the value type `T`. Populated automatically by the builder + /// from the `Schema` it was constructed with. + SchemaInfo schema; +}; + + + + +template +class StreamConsumerBuilder; + +/** + * Ordered (per-key), broker-managed consumer with **cumulative ack only**. + * + * This is the closest analog to a classic Failover subscription, but spanning all + * segments of a scalable topic (spec §7.1): messages are delivered in order + * (per-key) and the broker manages segment assignment for you. A single + * `acknowledgeCumulative` advances every segment up to the delivered message's + * position; there is no individual ack, no negative ack, and no dead-letter + * support. For parallel, unordered consumption with per-message ack use + * `QueueConsumer` instead. + * + * Obtain an instance from `StreamConsumerBuilder`. The default-constructed + * consumer is an empty handle (`operator bool` is `false`) until assigned one. + * + * @tparam T the decoded message value type, determined by the `Schema` used to + * build this consumer. + */ +template +class StreamConsumer { + public: + /** Construct an empty, non-live handle. `operator bool` returns `false` until a + * subscribed consumer is move-assigned into it. */ + StreamConsumer() = default; + + /** + * Block until the next message arrives and return it. + * + * Returns `Expected` because a receive can fail *without* yielding a message — + * the consumer was closed, the connection dropped, or the payload failed to + * decode. On such failures the result holds an `Error` instead of a message; + * call `.value()` on the result if you would rather throw a `ClientException`. + * + * @return the next `Message`, or an `Error` describing why no message could + * be delivered. + */ + Expected> receive() { return toTyped(core_.receiveAsync().get()); } + /** + * Block for the next message, but for no longer than `timeout`. + * + * @param timeout maximum time to wait for a message. + * @return the next `Message`; if no message arrives within `timeout`, an + * `Error{ResultTimeout}`; or another `Error` on close/disconnect/decode + * failure. + */ + Expected> receive(std::chrono::milliseconds timeout) { + return toTyped(core_.receiveAsync(timeout.count()).get()); + } + /** + * Request the next message without blocking. + * + * @return a `Future>` completed with the message when one is + * available, or completed with an `Error` (via the future's + * `Expected` result) on close/disconnect/decode failure. + */ + Future> receiveAsync() { + Schema schema = schema_; + return core_.receiveAsync().thenApply( + [schema](const detail::MessageCore& core) { return Message(core, schema); }); + } + + /** + * Block for a batch of up to `maxMessages`, bounded by `timeout`. + * + * Returns as soon as `maxMessages` have been collected or `timeout` elapses, + * whichever comes first, so the returned batch may contain fewer than + * `maxMessages` (including zero on timeout). + * + * @param maxMessages the maximum number of messages to return in the batch. + * @param timeout maximum time to wait while accumulating the batch. + * @return the collected `Messages`, or an `Error` on close/disconnect/decode + * failure. + */ + Expected> receiveMulti(int maxMessages, std::chrono::milliseconds timeout) { + return toTypedBatch(core_.receiveMultiAsync(maxMessages, timeout.count()).get()); + } + + /** + * Cumulatively acknowledge every message up to and including `id`, advancing all + * segments up to that position (spec §7.1). + * + * Fire-and-forget: it does not block and does not report an error. Acks are + * buffered and delivered best-effort; a lost ack simply causes redelivery. + * + * @param id the message position up to which (inclusive) to acknowledge. + */ + void acknowledgeCumulative(const MessageId& id) { core_.acknowledgeCumulative(id); } + /** + * Transactional cumulative acknowledge: enlist the cumulative ack up to `id` in + * `txn`. + * + * The acknowledgment becomes effective only when `txn` commits; its outcome (and + * any error) surfaces at `Transaction::commit()`, not here. + * + * @param id the message position up to which (inclusive) to acknowledge. + * @param txn the transaction the acknowledgment is enlisted in. + */ + void acknowledgeCumulative(const MessageId& id, const Transaction& txn) { + core_.acknowledgeCumulative(id, txn); + } + + /** + * Close the consumer, releasing its broker-side resources. Blocking. + * + * @return an empty `Expected` on success, or an `Error` if the close + * failed. Call `.value()` to throw instead. + */ + Expected close() { return core_.closeAsync().get(); } + /** + * Close the consumer without blocking. + * + * @return a `Future` completed when the close finishes (or with an `Error` + * on failure). + */ + Future closeAsync() { return core_.closeAsync(); } + + /** @return the topic this consumer is subscribed to. In namespace mode this is the + * namespace-derived subscription target. */ + const std::string& topic() const { return core_.topic(); } + /** @return the subscription name. */ + const std::string& subscription() const { return core_.subscription(); } + /** @return the consumer name (broker-assigned if none was set on the builder). */ + const std::string& consumerName() const { return core_.consumerName(); } + + /** @return `true` if this is a live, subscribed consumer; `false` if it is an empty + * (default-constructed or closed/moved-from) handle. */ + explicit operator bool() const { return static_cast(core_); } + + private: + template + friend class StreamConsumerBuilder; + StreamConsumer(detail::StreamConsumerCore core, Schema schema) + : core_(std::move(core)), schema_(std::move(schema)) {} + + Expected> toTyped(Expected r) const { + if (r) return Message(*r, schema_); + return Expected>(r.error()); + } + Expected> toTypedBatch(Expected> r) const { + if (!r) return Expected>(r.error()); + std::vector> out; + out.reserve(r->size()); + for (auto& core : *r) out.emplace_back(core, schema_); + return Messages(std::move(out)); + } + + detail::StreamConsumerCore core_; + Schema schema_; +}; + +/** + * Fluent builder for a `StreamConsumer`. + * + * Obtain one from `PulsarClient`. Set **exactly one** of `topic()` or + * `inNamespace()` to choose the subscription target, set the REQUIRED + * `subscriptionName()`, then call `subscribe()` / `subscribeAsync()` to create the + * consumer. All setters return `*this` for chaining. + * + * @tparam T the decoded message value type, fixed by the `Schema` the builder + * was created with. + */ +template +class StreamConsumerBuilder { + public: + /** + * Subscribe to a single scalable topic. Mutually exclusive with `inNamespace()`; + * set exactly one. Calling this clears any namespace selection. + * + * @param t the fully-qualified topic name. + * @return `*this` for chaining. + */ + StreamConsumerBuilder& topic(std::string t) { + config_.useNamespace = false; + config_.topic = std::move(t); + return *this; + } + + + /** + * Subscribe to all scalable topics in a namespace with live membership — topics + * created or removed later are joined/dropped automatically (spec §7.1). + * Mutually exclusive with `topic()`; set exactly one. + * + * Named `inNamespace` because `namespace` is a C++ keyword. + * + * @param ns the namespace (`tenant/namespace`) to subscribe across. + * @param propertyFilters optional AND filters matched against topic properties; + * only topics matching all entries are included. Default empty (no + * filtering — every topic in the namespace). + * @return `*this` for chaining. + */ + StreamConsumerBuilder& inNamespace(std::string ns, Properties propertyFilters = {}) { + config_.useNamespace = true; + config_.namespaceName = std::move(ns); + config_.propertyFilters = std::move(propertyFilters); + return *this; + } + /** + * REQUIRED. Set the subscription name shared by all consumers of this + * subscription. + * + * @param s the subscription name. + * @return `*this` for chaining. + */ + StreamConsumerBuilder& subscriptionName(std::string s) { + config_.subscriptionName = std::move(s); + return *this; + } + /** + * Set where the subscription starts when first created. + * + * @param p the initial position. Default `SubscriptionInitialPosition::Latest`. + * Has no effect if the subscription already exists. + * @return `*this` for chaining. + */ + StreamConsumerBuilder& subscriptionInitialPosition(SubscriptionInitialPosition p) { + config_.initialPosition = p; + return *this; + } + /** + * Attach key/value properties to the subscription itself (persisted broker-side). + * StreamConsumer only. + * + * @param p the subscription properties. Default empty. + * @return `*this` for chaining. + */ + StreamConsumerBuilder& subscriptionProperties(Properties p) { + config_.subscriptionProperties = std::move(p); + return *this; + } + /** + * Set an explicit consumer name (useful for diagnostics and metrics). + * + * @param n the consumer name. Default unset, in which case the broker assigns one. + * @return `*this` for chaining. + */ + StreamConsumerBuilder& consumerName(std::string n) { + config_.consumerName = std::move(n); + return *this; + } + + /** + * Tune acknowledgment behavior (e.g. the ack-grouping/batching window). + * + * @param policy the ack policy. Default-constructed `AckPolicy` when unset. + * @return `*this` for chaining. + */ + StreamConsumerBuilder& ackPolicy(AckPolicy policy) { + config_.ackPolicy = std::move(policy); + return *this; + } + /** + * Read the topic's compacted view (latest value per key) instead of the full log. + * StreamConsumer only. + * + * @param b `true` to read compacted. Default unset (broker default: uncompacted). + * @return `*this` for chaining. + */ + StreamConsumerBuilder& readCompacted(bool b) { + config_.readCompacted = b; + return *this; + } + /** + * Replicate the subscription's acknowledged position across geo-replication + * clusters. StreamConsumer only. + * + * @param b `true` to enable. Default unset (broker default: disabled). + * @return `*this` for chaining. + */ + StreamConsumerBuilder& replicateSubscriptionState(bool b) { + config_.replicateSubscriptionState = b; + return *this; + } + /** + * Add a single client-side consumer property (reported in topic stats). Call + * repeatedly to add multiple; a repeated key overwrites the previous value. + * + * @param k property key. + * @param v property value. + * @return `*this` for chaining. + */ + StreamConsumerBuilder& property(const std::string& k, const std::string& v) { + config_.properties[k] = v; + return *this; + } + + /** + * Create the consumer and subscribe. Blocking. + * + * @return the live `StreamConsumer` on success, or an `Error` if the + * subscription failed (e.g. missing `subscriptionName`, both/neither of + * topic and namespace set, or a broker error). Call `.value()` to throw + * instead. + */ + Expected> subscribe() { return subscribeAsync().get(); } + /** + * Create the consumer and subscribe without blocking. + * + * @return a `Future>` completed with the live consumer, or with + * an `Error` on failure. + */ + Future> subscribeAsync() { + Schema schema = schema_; + StreamConsumerConfig config = config_; + config.schema = schema.info(); + return client_.subscribeStreamAsync(std::move(config)) + .thenApply([schema](const detail::StreamConsumerCore& core) { return StreamConsumer(core, schema); }); + } + + private: + friend class PulsarClient; + StreamConsumerBuilder(detail::ClientCore client, Schema schema) + : client_(std::move(client)), schema_(std::move(schema)) {} + + detail::ClientCore client_; + Schema schema_; + StreamConsumerConfig config_; +}; + +} // namespace pulsar::st diff --git a/include/pulsar/st/Transaction.h b/include/pulsar/st/Transaction.h new file mode 100644 index 00000000..0b69ba5b --- /dev/null +++ b/include/pulsar/st/Transaction.h @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include + +#include + +namespace pulsar::st { + +class TransactionImpl; +using TransactionImplPtr = std::shared_ptr; +namespace detail { +class ClientCore; +class ProducerCore; +class StreamConsumerCore; +class QueueConsumerCore; +} // namespace detail + +/** + * @brief Lifecycle states of a transaction (spec §9). + * + * A transaction starts `Open`, transitions through a transient `Committing` or + * `Aborting` phase while the outcome is being made durable, and ends in one of the + * terminal states `Committed`, `Aborted`, `Error`, or `TimedOut`. Query the + * current state with `Transaction::state()`. + */ +enum class TransactionState { + Open, ///< Active: messages and acks may still be enlisted; not yet committed or aborted. + Committing, ///< Transient: a `commit()`/`commitAsync()` is in progress but not yet durable. + Aborting, ///< Transient: an `abort()`/`abortAsync()` is in progress but not yet finalized. + Committed, ///< Terminal: committed successfully; produced messages are visible and acks durable. + Aborted, ///< Terminal: aborted; produced messages are discarded and acks rolled back. + Error, ///< Terminal: a failure left the transaction in an unrecoverable state. + TimedOut, ///< Terminal: the transaction timeout elapsed before commit, so it was aborted. +}; + +/** + * @brief A transaction providing exactly-once semantics across multiple scalable + * topics and subscriptions (spec §9). + * + * Messages produced and acknowledgments made within a single transaction are + * applied atomically: on commit they all take effect, and on abort none of them + * do. This lets an application consume, transform, and produce across topics with + * no duplicates and no lost work. + * + * Typical usage: + * -# obtain a transaction from `PulsarClient::newTransaction()`; + * -# enlist publishes with `MessageBuilder::transaction(txn)` and acknowledgments + * with `consumer.acknowledge*(id, txn)`; + * -# call `commit()` to make the produced messages visible and the acks durable, + * or `abort()` to discard everything done within the transaction. + * + * A default-constructed `Transaction` is empty (falsy under `operator bool`) and + * must not be used to enlist or commit work. + * + * Holds the hidden `TransactionImpl`; its operations are defined in lib/st. + */ +class PULSAR_PUBLIC Transaction { + public: + /** @brief Construct an empty, unusable transaction (falsy under `operator bool`). */ + Transaction() = default; + + /** + * @brief Return the current lifecycle state of this transaction. + * + * @return The current `TransactionState`. + */ + TransactionState state() const; + + /** + * @brief Commit the transaction: make produced messages visible and acks durable. + * + * Atomically applies every publish and acknowledgment enlisted in this + * transaction. Blocks until the outcome is durable. + * + * @return `Expected` holding success, or an `Error` if the commit failed. + */ + Expected commit() { return commitAsync().get(); } + + /** + * @brief Asynchronously commit the transaction. + * + * Non-blocking counterpart of `commit()`; the returned future completes when the + * commit is durable or fails. + * + * @return `Future` resolving to success, or an `Error` on failure. + */ + Future commitAsync() const; + + /** + * @brief Abort the transaction: discard produced messages and roll back acks. + * + * Atomically discards every publish and acknowledgment enlisted in this + * transaction, as if none had happened. Blocks until the abort is finalized. + * + * @return `Expected` holding success, or an `Error` if the abort failed. + */ + Expected abort() { return abortAsync().get(); } + + /** + * @brief Asynchronously abort the transaction. + * + * Non-blocking counterpart of `abort()`; the returned future completes when the + * abort is finalized or fails. + * + * @return `Future` resolving to success, or an `Error` on failure. + */ + Future abortAsync() const; + + /** + * @brief Test whether this object wraps a live transaction. + * + * @return `true` for a transaction obtained from `PulsarClient::newTransaction()`; + * `false` for a default-constructed (empty) one. + */ + explicit operator bool() const { return static_cast(impl_); } + + private: + // The client constructs a Transaction; the producer/consumer cores read its + // impl to enlist sends and acks. All of these live in lib/st. + friend class detail::ClientCore; + friend class detail::ProducerCore; + friend class detail::StreamConsumerCore; + friend class detail::QueueConsumerCore; + explicit Transaction(TransactionImplPtr impl) : impl_(std::move(impl)) {} + + TransactionImplPtr impl_; +}; + +} // namespace pulsar::st diff --git a/include/pulsar/st/detail/CheckpointConsumerCore.h b/include/pulsar/st/detail/CheckpointConsumerCore.h new file mode 100644 index 00000000..9b3a67c7 --- /dev/null +++ b/include/pulsar/st/detail/CheckpointConsumerCore.h @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace pulsar::st { + +class CheckpointConsumerImpl; +using CheckpointConsumerImplPtr = std::shared_ptr; + +namespace detail { + +class ClientCore; + +/** + * INTERNAL — not part of the public API. Non-templated checkpoint-consumer + * operations over the hidden impl (lib/st). `CheckpointConsumer` wraps it. + */ +class PULSAR_PUBLIC CheckpointConsumerCore { + public: + CheckpointConsumerCore() = default; + + Future receiveAsync() const; + Future receiveAsync(int64_t timeoutMs) const; + Future> receiveMultiAsync(int maxMessages, int64_t timeoutMs) const; + Checkpoint checkpoint() const; + Future closeAsync() const; + const std::string& topic() const; + + explicit operator bool() const { return static_cast(impl_); } + + private: + friend class ClientCore; + explicit CheckpointConsumerCore(CheckpointConsumerImplPtr impl) : impl_(std::move(impl)) {} + + CheckpointConsumerImplPtr impl_; +}; + +} // namespace detail +} // namespace pulsar::st diff --git a/include/pulsar/st/detail/ClientCore.h b/include/pulsar/st/detail/ClientCore.h new file mode 100644 index 00000000..e7c957ab --- /dev/null +++ b/include/pulsar/st/detail/ClientCore.h @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include + +#include + +namespace pulsar::st { + +class ClientImpl; +using ClientImplPtr = std::shared_ptr; +class Transaction; +class PulsarClientBuilder; +struct ProducerConfig; +struct StreamConsumerConfig; +struct QueueConsumerConfig; +struct CheckpointConsumerConfig; + +namespace detail { + +class ProducerCore; +class StreamConsumerCore; +class QueueConsumerCore; +class CheckpointConsumerCore; + +/** + * INTERNAL — not part of the public API. The non-templated client operations that + * the templated builders call across to reach the core in lib/st (the typed-API + * equivalent of the methods today's non-templated `Client` defines out-of-line). + * Holds the hidden `ClientImpl`; applications use `PulsarClient`, not this. + */ +class PULSAR_PUBLIC ClientCore { + public: + ClientCore() = default; + + Future createProducerAsync(ProducerConfig config) const; + Future subscribeStreamAsync(StreamConsumerConfig config) const; + Future subscribeQueueAsync(QueueConsumerConfig config) const; + Future createCheckpointAsync(CheckpointConsumerConfig config) const; + Future newTransactionAsync() const; + Future closeAsync() const; + void shutdown() const; + + explicit operator bool() const { return static_cast(impl_); } + + private: + friend class pulsar::st::PulsarClientBuilder; + explicit ClientCore(ClientImplPtr impl) : impl_(std::move(impl)) {} + + ClientImplPtr impl_; +}; + +} // namespace detail +} // namespace pulsar::st diff --git a/include/pulsar/st/detail/Cxx20.h b/include/pulsar/st/detail/Cxx20.h new file mode 100644 index 00000000..1fae2385 --- /dev/null +++ b/include/pulsar/st/detail/Cxx20.h @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +// The pulsar::st (scalable topics) API targets C++20. The rest of the Pulsar C++ +// client remains C++17 — only this new API requires C++20 (for concepts, +// coroutine-awaitable Future, `using enum`, reflection-based schemas, etc.). +#if (defined(_MSVC_LANG) ? _MSVC_LANG : __cplusplus) < 202002L +#error "pulsar::st (scalable topics) requires C++20. Build this translation unit with -std=c++20 (or /std:c++20)." +#endif diff --git a/include/pulsar/st/detail/MessageCore.h b/include/pulsar/st/detail/MessageCore.h new file mode 100644 index 00000000..3c32ccdd --- /dev/null +++ b/include/pulsar/st/detail/MessageCore.h @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace pulsar::st { + +class MessageImpl; +using MessageImplPtr = std::shared_ptr; + +using Timestamp = std::chrono::system_clock::time_point; +using Properties = std::map; + +namespace detail { + +class StreamConsumerCore; +class QueueConsumerCore; +class CheckpointConsumerCore; + +/** + * INTERNAL — not part of the public API. Non-templated, byte-oriented view of a + * received message; its accessors are defined in lib/st. `Message` wraps this + * and decodes the payload through `Schema`. + */ +class PULSAR_PUBLIC MessageCore { + public: + MessageCore() = default; + + const char* data() const; + std::size_t size() const; + MessageId id() const; + bool hasKey() const; + const std::string& key() const; + const Properties& properties() const; + int64_t publishTimeMs() const; + int64_t eventTimeMs() const; // 0 if unset + int64_t sequenceId() const; + bool hasProducerName() const; + const std::string& producerName() const; + const std::string& topic() const; + int redeliveryCount() const; + bool hasReplicatedFrom() const; + const std::string& replicatedFrom() const; + + explicit operator bool() const { return static_cast(impl_); } + + private: + friend class StreamConsumerCore; + friend class QueueConsumerCore; + friend class CheckpointConsumerCore; + explicit MessageCore(MessageImplPtr impl) : impl_(std::move(impl)) {} + + MessageImplPtr impl_; +}; + +} // namespace detail +} // namespace pulsar::st diff --git a/include/pulsar/st/detail/ProducerCore.h b/include/pulsar/st/detail/ProducerCore.h new file mode 100644 index 00000000..543eb750 --- /dev/null +++ b/include/pulsar/st/detail/ProducerCore.h @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace pulsar::st { + +class ProducerImplBase; +using ProducerImplPtr = std::shared_ptr; +struct OutgoingMessage; + +namespace detail { + +class ClientCore; + +/** + * INTERNAL — not part of the public API. Non-templated producer operations over + * the hidden `ProducerImpl` (defined in lib/st). `Producer` / `MessageBuilder` + * are thin wrappers over it. + */ +class PULSAR_PUBLIC ProducerCore { + public: + ProducerCore() = default; + + Future sendAsync(OutgoingMessage message) const; + const std::string& topic() const; + const std::string& name() const; + int64_t lastSequenceId() const; + Future flushAsync() const; + Future closeAsync() const; + + explicit operator bool() const { return static_cast(impl_); } + + private: + friend class ClientCore; + explicit ProducerCore(ProducerImplPtr impl) : impl_(std::move(impl)) {} + + ProducerImplPtr impl_; +}; + +} // namespace detail +} // namespace pulsar::st diff --git a/include/pulsar/st/detail/QueueConsumerCore.h b/include/pulsar/st/detail/QueueConsumerCore.h new file mode 100644 index 00000000..ef8e0035 --- /dev/null +++ b/include/pulsar/st/detail/QueueConsumerCore.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace pulsar::st { + +class QueueConsumerImpl; +using QueueConsumerImplPtr = std::shared_ptr; +class Transaction; + +namespace detail { + +class ClientCore; + +/** + * INTERNAL — not part of the public API. Non-templated queue-consumer operations + * over the hidden impl (lib/st). `QueueConsumer` is a thin wrapper over it. + */ +class PULSAR_PUBLIC QueueConsumerCore { + public: + QueueConsumerCore() = default; + + Future receiveAsync() const; + Future receiveAsync(int64_t timeoutMs) const; + void acknowledge(const MessageId& id) const; + void acknowledge(const MessageId& id, const Transaction& txn) const; + void negativeAcknowledge(const MessageId& id) const; + Future closeAsync() const; + const std::string& topic() const; + const std::string& subscription() const; + const std::string& consumerName() const; + + explicit operator bool() const { return static_cast(impl_); } + + private: + friend class ClientCore; + explicit QueueConsumerCore(QueueConsumerImplPtr impl) : impl_(std::move(impl)) {} + + QueueConsumerImplPtr impl_; +}; + +} // namespace detail +} // namespace pulsar::st diff --git a/include/pulsar/st/detail/SharedState.h b/include/pulsar/st/detail/SharedState.h new file mode 100644 index 00000000..ccbfe4a7 --- /dev/null +++ b/include/pulsar/st/detail/SharedState.h @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +// INTERNAL. The set-once completion state behind Future/Promise. It lives +// in pulsar::st::detail and is not part of the public API; applications use +// Future only. It must sit in a header because Future/Promise are templates. + +namespace pulsar::st::detail { + +template +class SharedState { + public: + using Listener = std::function&)>; + + bool complete(Expected result) { + std::unique_lock lock(mutex_); + if (result_.has_value()) { + return false; + } + result_.emplace(std::move(result)); + cond_.notify_all(); + std::vector listeners = std::move(listeners_); + listeners_.clear(); + lock.unlock(); + for (auto& listener : listeners) { + listener(*result_); + } + return true; + } + + void addListener(Listener listener) { + std::unique_lock lock(mutex_); + if (result_.has_value()) { + Expected snapshot = *result_; + lock.unlock(); + listener(snapshot); + } else { + listeners_.push_back(std::move(listener)); + } + } + + Expected get() { + std::unique_lock lock(mutex_); + cond_.wait(lock, [this] { return result_.has_value(); }); + return *result_; + } + + template + std::optional> get(std::chrono::duration timeout) { + std::unique_lock lock(mutex_); + if (!cond_.wait_for(lock, timeout, [this] { return result_.has_value(); })) { + return std::nullopt; + } + return *result_; + } + + bool isReady() const { + std::lock_guard lock(mutex_); + return result_.has_value(); + } + + private: + mutable std::mutex mutex_; + std::condition_variable cond_; + std::optional> result_; + std::vector listeners_; +}; + +} // namespace pulsar::st::detail diff --git a/include/pulsar/st/detail/StreamConsumerCore.h b/include/pulsar/st/detail/StreamConsumerCore.h new file mode 100644 index 00000000..4952d7ba --- /dev/null +++ b/include/pulsar/st/detail/StreamConsumerCore.h @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace pulsar::st { + +class StreamConsumerImpl; +using StreamConsumerImplPtr = std::shared_ptr; +class Transaction; + +namespace detail { + +class ClientCore; + +/** + * INTERNAL — not part of the public API. Non-templated stream-consumer operations + * over the hidden impl (lib/st). `StreamConsumer` is a thin wrapper over it. + */ +class PULSAR_PUBLIC StreamConsumerCore { + public: + StreamConsumerCore() = default; + + Future receiveAsync() const; + Future receiveAsync(int64_t timeoutMs) const; + Future> receiveMultiAsync(int maxMessages, int64_t timeoutMs) const; + void acknowledgeCumulative(const MessageId& id) const; + void acknowledgeCumulative(const MessageId& id, const Transaction& txn) const; + Future closeAsync() const; + const std::string& topic() const; + const std::string& subscription() const; + const std::string& consumerName() const; + + explicit operator bool() const { return static_cast(impl_); } + + private: + friend class ClientCore; + explicit StreamConsumerCore(StreamConsumerImplPtr impl) : impl_(std::move(impl)) {} + + StreamConsumerImplPtr impl_; +}; + +} // namespace detail +} // namespace pulsar::st diff --git a/vcpkg.json b/vcpkg.json index 3452492d..721371da 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -43,6 +43,10 @@ "name": "protobuf", "version>=": "6.33.4#1" }, + { + "name": "reflectcpp", + "version>=": "0.24.0" + }, { "name": "snappy", "version>=": "1.2.2" From fa4def33d7d427983052e3553b6c254f5d8ac77d Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 23 Jun 2026 12:28:38 -0700 Subject: [PATCH 2/3] Fix CI: clang-format the st sources; make reflect-cpp optional - Apply clang-format-11 to the new pulsar::st headers and examples (the Formatting Check uses clang-format 11; local 18 formats differently). - examples/CMakeLists.txt: build the four dependency-free st samples unconditionally and add the reflect-cpp JSON sample only when reflectcpp is found (find_package CONFIG QUIET instead of REQUIRED), so configure no longer fails where reflect-cpp is absent (e.g. the CodeQL/Analyze job). - vcpkg.json: drop the reflectcpp dependency for now; it returns with the lib/st implementation that actually exercises the JSON/Avro schemas. Signed-off-by: Matteo Merli --- examples/CMakeLists.txt | 25 ++++++++++++------- examples/st/SampleStProducer.cc | 7 ++++-- examples/st/SampleStQueueConsumer.cc | 2 +- include/pulsar/st/AvroSchema.h | 3 +-- include/pulsar/st/CheckpointConsumer.h | 21 ++++++++-------- include/pulsar/st/Client.h | 2 +- include/pulsar/st/Consumer.h | 15 ++++++++---- include/pulsar/st/Error.h | 3 ++- include/pulsar/st/Expected.h | 3 +-- include/pulsar/st/Future.h | 2 +- include/pulsar/st/JsonSchema.h | 3 +-- include/pulsar/st/Message.h | 3 ++- include/pulsar/st/Policies.h | 33 +++++++++++++++++--------- include/pulsar/st/Producer.h | 21 ++++++++-------- include/pulsar/st/QueueConsumer.h | 11 ++++----- include/pulsar/st/Schema.h | 14 +++++++---- include/pulsar/st/StreamConsumer.h | 12 ++++------ include/pulsar/st/Transaction.h | 3 ++- include/pulsar/st/detail/Cxx20.h | 3 ++- vcpkg.json | 4 ---- 20 files changed, 106 insertions(+), 84 deletions(-) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 81796c09..f0c0db2f 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -116,22 +116,31 @@ target_link_libraries(SampleCustomLoggerCApi ${CLIENT_LIBS} pulsarShar # TODO(scalable-topics): once lib/st lands, replace this with one # add_executable + target_link_libraries(... pulsarShared) per file, exactly like # the samples above. +# The core samples are header-only previews of the pulsar::st API and build with +# no extra dependency. set(SAMPLE_ST_SOURCES st/SampleStProducer.cc st/SampleStStreamConsumer.cc st/SampleStQueueConsumer.cc st/SampleStCheckpointConsumer.cc - st/SampleStJsonSchema.cc ) -# reflect-cpp powers jsonSchema() (reflection-based JSON SerDe + schema) and is -# a required dependency of the scalable-topics API. -find_package(reflectcpp CONFIG REQUIRED) +# reflect-cpp powers jsonSchema() (reflection-based JSON SerDe + schema). It is +# optional for this API-only PR: when the package is present the JSON sample is +# added and linked against it; when absent, only that one sample is skipped. (The +# reflectcpp vcpkg port does not yet ship an Avro backend, so it is not yet wired +# into the manifest; it will be added with the lib/st implementation.) +find_package(reflectcpp CONFIG QUIET) +if (reflectcpp_FOUND) + list(APPEND SAMPLE_ST_SOURCES st/SampleStJsonSchema.cc) +endif () add_library(StExamples OBJECT ${SAMPLE_ST_SOURCES}) # The scalable-topics (pulsar::st) API targets C++20; the rest of the client stays # C++17. Set the standard per-target so only this code requires C++20. set_target_properties(StExamples PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED ON) -# PRIVATE link gives the object sources pulsarShared's and reflect-cpp's include -# directories; an OBJECT library is not itself linked, so the missing lib/st -# symbols are fine. -target_link_libraries(StExamples PRIVATE ${CLIENT_LIBS} pulsarShared reflectcpp::reflectcpp) +# PRIVATE link gives the object sources pulsarShared's include directories; an +# OBJECT library is not itself linked, so the missing lib/st symbols are fine. +target_link_libraries(StExamples PRIVATE ${CLIENT_LIBS} pulsarShared) +if (reflectcpp_FOUND) + target_link_libraries(StExamples PRIVATE reflectcpp::reflectcpp) +endif () diff --git a/examples/st/SampleStProducer.cc b/examples/st/SampleStProducer.cc index 2252c3dd..8f7431a1 100644 --- a/examples/st/SampleStProducer.cc +++ b/examples/st/SampleStProducer.cc @@ -59,8 +59,11 @@ int main() { } // Asynchronous send: react on completion without blocking. - producer.newMessage().key("order-async").value("async-payload").sendAsync().addListener( - [](const Expected& result) { + producer.newMessage() + .key("order-async") + .value("async-payload") + .sendAsync() + .addListener([](const Expected& result) { if (result) { std::cout << "async sent " << *result << "\n"; } else { diff --git a/examples/st/SampleStQueueConsumer.cc b/examples/st/SampleStQueueConsumer.cc index f8171440..1efac129 100644 --- a/examples/st/SampleStQueueConsumer.cc +++ b/examples/st/SampleStQueueConsumer.cc @@ -56,7 +56,7 @@ int main() { const bool processed = !msg->value().empty(); if (processed) { - consumer.acknowledge(msg->id()); // fire-and-forget; never blocks or errors + consumer.acknowledge(msg->id()); // fire-and-forget; never blocks or errors } else { consumer.negativeAcknowledge(msg->id()); // schedule redelivery } diff --git a/include/pulsar/st/AvroSchema.h b/include/pulsar/st/AvroSchema.h index a0ac13eb..ccf88656 100644 --- a/include/pulsar/st/AvroSchema.h +++ b/include/pulsar/st/AvroSchema.h @@ -20,10 +20,9 @@ #include +#include #include #include - -#include #include // avroSchema() is the Avro counterpart of jsonSchema(): reflect-cpp derives diff --git a/include/pulsar/st/CheckpointConsumer.h b/include/pulsar/st/CheckpointConsumer.h index e5bb80d7..8f063706 100644 --- a/include/pulsar/st/CheckpointConsumer.h +++ b/include/pulsar/st/CheckpointConsumer.h @@ -20,12 +20,12 @@ #include #include -#include -#include #include #include #include #include +#include +#include #include #include @@ -36,7 +36,6 @@ namespace pulsar::st { - /** * @brief Configuration accumulated by `CheckpointConsumerBuilder`. * @@ -46,16 +45,16 @@ namespace pulsar::st { */ struct CheckpointConsumerConfig { std::string topic; ///< Scalable topic to read. REQUIRED; no default. - Checkpoint startPosition = Checkpoint::latest(); ///< Position to start from. Default `Checkpoint::latest()`. - std::optional consumerGroup; ///< Consumer group to join. Unset (default) => ungrouped, reads every segment. - std::optional consumerName; ///< Human-readable consumer name. Unset (default) => auto-generated. - Properties properties; ///< Free-form key/value metadata attached to the consumer. Default empty. - SchemaInfo schema; ///< Schema descriptor; filled in from `Schema` by the builder. + Checkpoint startPosition = + Checkpoint::latest(); ///< Position to start from. Default `Checkpoint::latest()`. + std::optional + consumerGroup; ///< Consumer group to join. Unset (default) => ungrouped, reads every segment. + std::optional + consumerName; ///< Human-readable consumer name. Unset (default) => auto-generated. + Properties properties; ///< Free-form key/value metadata attached to the consumer. Default empty. + SchemaInfo schema; ///< Schema descriptor; filled in from `Schema` by the builder. }; - - - template class CheckpointConsumerBuilder; diff --git a/include/pulsar/st/Client.h b/include/pulsar/st/Client.h index 3cd82529..be0eedc4 100644 --- a/include/pulsar/st/Client.h +++ b/include/pulsar/st/Client.h @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -30,6 +29,7 @@ #include #include #include +#include #include #include diff --git a/include/pulsar/st/Consumer.h b/include/pulsar/st/Consumer.h index 81b430b9..8c346656 100644 --- a/include/pulsar/st/Consumer.h +++ b/include/pulsar/st/Consumer.h @@ -38,7 +38,8 @@ namespace pulsar::st { * created. It is ignored once the subscription exists and has a durable cursor: * an already-established subscription always resumes from its stored position. */ -enum class SubscriptionInitialPosition { +enum class SubscriptionInitialPosition +{ Earliest, ///< Start from the oldest available message on the topic. Latest ///< Start from the newest message, skipping anything published before subscribing. }; @@ -51,9 +52,11 @@ enum class SubscriptionInitialPosition { * optional and fall back to the client default when unset. */ struct AckPolicy { - /** Time window over which acknowledgments are batched before being sent, in milliseconds; 0 acks immediately. Unset uses the client default. */ + /** Time window over which acknowledgments are batched before being sent, in milliseconds; 0 acks + * immediately. Unset uses the client default. */ std::optional groupTime; - /** Delay before a negatively-acknowledged message is redelivered, in milliseconds. QueueConsumer only. Unset uses the client default. */ + /** Delay before a negatively-acknowledged message is redelivered, in milliseconds. QueueConsumer only. + * Unset uses the client default. */ std::optional negativeAckRedeliveryDelay; }; @@ -66,11 +69,13 @@ struct AckPolicy { * to a positive value. */ struct DeadLetterPolicy { - /** Maximum number of redeliveries before a message is routed to the dead-letter topic. Defaults to 0, which disables dead-lettering. */ + /** Maximum number of redeliveries before a message is routed to the dead-letter topic. Defaults to 0, + * which disables dead-lettering. */ int maxRedeliverCount = 0; /** Name of the dead-letter topic. Unset defaults to "<topic>-<subscription>-DLQ". */ std::optional deadLetterTopic; - /** If set, creates this subscription on the dead-letter topic up front so no messages are missed before a consumer attaches. Unset creates no initial subscription. */ + /** If set, creates this subscription on the dead-letter topic up front so no messages are missed before a + * consumer attaches. Unset creates no initial subscription. */ std::optional initialSubscriptionName; }; diff --git a/include/pulsar/st/Error.h b/include/pulsar/st/Error.h index 9e90e016..bc9bd97a 100644 --- a/include/pulsar/st/Error.h +++ b/include/pulsar/st/Error.h @@ -38,7 +38,8 @@ namespace pulsar::st { using pulsar::Error; /** Re-export of `pulsar::Result`: the enumeration of machine-readable result codes. */ using pulsar::Result; -/** Re-export the `Result` enumerators into `pulsar::st` so they are usable unqualified (e.g. `ResultTimeout`). */ +/** Re-export the `Result` enumerators into `pulsar::st` so they are usable unqualified (e.g. + * `ResultTimeout`). */ using enum pulsar::Result; /** diff --git a/include/pulsar/st/Expected.h b/include/pulsar/st/Expected.h index 05871493..3ea2f5cb 100644 --- a/include/pulsar/st/Expected.h +++ b/include/pulsar/st/Expected.h @@ -216,8 +216,7 @@ class [[nodiscard]] Expected { template auto transform(F&& f) const& { using U = std::remove_cv_t>>; - return has_value() ? Expected(std::forward(f)(std::get<0>(storage_))) - : Expected(error()); + return has_value() ? Expected(std::forward(f)(std::get<0>(storage_))) : Expected(error()); } /** diff --git a/include/pulsar/st/Future.h b/include/pulsar/st/Future.h index 000f06f9..8e2f6c26 100644 --- a/include/pulsar/st/Future.h +++ b/include/pulsar/st/Future.h @@ -23,9 +23,9 @@ #include #include +#include #include #include -#include #include #include diff --git a/include/pulsar/st/JsonSchema.h b/include/pulsar/st/JsonSchema.h index 03af1a22..15fa6987 100644 --- a/include/pulsar/st/JsonSchema.h +++ b/include/pulsar/st/JsonSchema.h @@ -20,10 +20,9 @@ #include +#include #include #include - -#include #include // jsonSchema() derives BOTH the JSON SerDe and the declared schema from T's diff --git a/include/pulsar/st/Message.h b/include/pulsar/st/Message.h index 055ba7a7..f029521c 100644 --- a/include/pulsar/st/Message.h +++ b/include/pulsar/st/Message.h @@ -55,7 +55,8 @@ class Message { * @param core the raw received message (payload and metadata). * @param schema the schema used to decode the payload in `value()`. */ - Message(detail::MessageCore core, Schema schema) : core_(std::move(core)), schema_(std::move(schema)) {} + Message(detail::MessageCore core, Schema schema) + : core_(std::move(core)), schema_(std::move(schema)) {} /** * Decode the payload through `Schema` and return the typed value. diff --git a/include/pulsar/st/Policies.h b/include/pulsar/st/Policies.h index 5ffb44ba..29c0d739 100644 --- a/include/pulsar/st/Policies.h +++ b/include/pulsar/st/Policies.h @@ -90,17 +90,22 @@ struct MemorySize { struct ConnectionPolicy { /** Number of physical connections opened to each broker. Unset uses the client default. */ std::optional connectionsPerBroker; - /** Maximum time to wait for a TCP/TLS connection to be established, in milliseconds. Unset uses the client default. */ + /** Maximum time to wait for a TCP/TLS connection to be established, in milliseconds. Unset uses the + * client default. */ std::optional connectionTimeout; - /** Maximum time to wait for a broker request (e.g. produce/consume control ops) to complete, in milliseconds. Unset uses the client default. */ + /** Maximum time to wait for a broker request (e.g. produce/consume control ops) to complete, in + * milliseconds. Unset uses the client default. */ std::optional operationTimeout; - /** Interval between keep-alive pings sent on an idle connection, in seconds. Unset uses the client default. */ + /** Interval between keep-alive pings sent on an idle connection, in seconds. Unset uses the client + * default. */ std::optional keepAliveInterval; /** Maximum number of concurrent topic-lookup requests in flight. Unset uses the client default. */ std::optional maxLookupRequests; - /** Maximum number of lookup redirects to follow before failing a lookup. Unset uses the client default. */ + /** Maximum number of lookup redirects to follow before failing a lookup. Unset uses the client default. + */ std::optional maxLookupRedirects; - /** Time an idle pooled connection may stay open before being closed, in milliseconds. Unset uses the client default. */ + /** Time an idle pooled connection may stay open before being closed, in milliseconds. Unset uses the + * client default. */ std::optional maxConnectionIdleTime; }; @@ -114,7 +119,8 @@ struct ConnectionPolicy { struct BackoffPolicy { /** Delay before the first reconnection attempt, in milliseconds. Unset uses the client default. */ std::optional initialBackoff; - /** Upper bound on the backoff delay as it grows across retries, in milliseconds. Unset uses the client default. */ + /** Upper bound on the backoff delay as it grows across retries, in milliseconds. Unset uses the client + * default. */ std::optional maxBackoff; }; @@ -128,13 +134,17 @@ struct BackoffPolicy { struct TlsPolicy { /** Whether TLS is used for broker connections. Defaults to false (plaintext). */ bool enabled = false; - /** Path to the PEM file of trusted CA certificates used to verify the broker. Unset uses the system trust store. */ + /** Path to the PEM file of trusted CA certificates used to verify the broker. Unset uses the system trust + * store. */ std::optional trustCertsFilePath; - /** Path to the client certificate PEM file, for mutual TLS. Unset disables client-certificate authentication. */ + /** Path to the client certificate PEM file, for mutual TLS. Unset disables client-certificate + * authentication. */ std::optional certificateFilePath; - /** Path to the client private key PEM file, for mutual TLS. Unset disables client-certificate authentication. */ + /** Path to the client private key PEM file, for mutual TLS. Unset disables client-certificate + * authentication. */ std::optional privateKeyFilePath; - /** Whether to accept the broker's certificate without validating it against the trust store. Defaults to false (validation enforced). */ + /** Whether to accept the broker's certificate without validating it against the trust store. Defaults to + * false (validation enforced). */ bool allowInsecureConnection = false; /** Whether to verify that the broker's certificate hostname matches the endpoint. Defaults to true. */ bool validateHostname = true; @@ -148,7 +158,8 @@ struct TlsPolicy { * optional and the client supplies a built-in default when it is unset. */ struct TransactionPolicy { - /** Default lifetime of a transaction before it is automatically aborted, in milliseconds. Unset uses the client default. */ + /** Default lifetime of a transaction before it is automatically aborted, in milliseconds. Unset uses the + * client default. */ std::optional timeout; }; diff --git a/include/pulsar/st/Producer.h b/include/pulsar/st/Producer.h index 0fe4fc10..be1774e6 100644 --- a/include/pulsar/st/Producer.h +++ b/include/pulsar/st/Producer.h @@ -19,8 +19,6 @@ #pragma once #include -#include -#include #include #include #include @@ -28,6 +26,8 @@ #include #include #include +#include +#include #include #include @@ -45,7 +45,8 @@ namespace pulsar::st { * and controls how the broker arbitrates between multiple producers on the same * topic. */ -enum class ProducerAccessMode { +enum class ProducerAccessMode +{ /** Multiple producers may publish to the topic concurrently. The default. */ Shared, /** Only one producer may be active at a time; another producer requesting @@ -106,9 +107,9 @@ struct OutgoingMessage { std::string key; /** Per-message user metadata. Empty by default. */ Properties properties; - int64_t eventTimeMs = 0; ///< Application event time, epoch ms; 0 = unset. - int64_t sequenceId = -1; ///< Explicit sequence id; -1 = auto-assign. - int64_t deliverAtMs = 0; ///< Absolute delivery time, epoch ms; 0 = deliver immediately. + int64_t eventTimeMs = 0; ///< Application event time, epoch ms; 0 = unset. + int64_t sequenceId = -1; ///< Explicit sequence id; -1 = auto-assign. + int64_t deliverAtMs = 0; ///< Absolute delivery time, epoch ms; 0 = deliver immediately. /** Target clusters for geo-replication; empty applies the topic's default. */ std::vector replicationClusters; std::optional transaction; ///< Enlisting transaction; unset = non-transactional. @@ -350,7 +351,8 @@ class Producer { private: template friend class ProducerBuilder; - Producer(detail::ProducerCore core, Schema schema) : core_(std::move(core)), schema_(std::move(schema)) {} + Producer(detail::ProducerCore core, Schema schema) + : core_(std::move(core)), schema_(std::move(schema)) {} detail::ProducerCore core_; Schema schema_; @@ -460,9 +462,8 @@ class ProducerBuilder { Schema schema = schema_; ProducerConfig config = config_; config.schema = schema.info(); - return client_.createProducerAsync(std::move(config)).thenApply([schema](const detail::ProducerCore& core) { - return Producer(core, schema); - }); + return client_.createProducerAsync(std::move(config)) + .thenApply([schema](const detail::ProducerCore& core) { return Producer(core, schema); }); } private: diff --git a/include/pulsar/st/QueueConsumer.h b/include/pulsar/st/QueueConsumer.h index c82f2256..42aae798 100644 --- a/include/pulsar/st/QueueConsumer.h +++ b/include/pulsar/st/QueueConsumer.h @@ -19,8 +19,6 @@ #pragma once #include -#include -#include #include #include #include @@ -28,6 +26,8 @@ #include #include #include +#include +#include #include #include @@ -38,7 +38,6 @@ namespace pulsar::st { - /** * Plain-old-data configuration accumulated by `QueueConsumerBuilder`. * @@ -83,9 +82,6 @@ struct QueueConsumerConfig { SchemaInfo schema; }; - - - template class QueueConsumerBuilder; @@ -356,7 +352,8 @@ class QueueConsumerBuilder { QueueConsumerConfig config = config_; config.schema = schema.info(); return client_.subscribeQueueAsync(std::move(config)) - .thenApply([schema](const detail::QueueConsumerCore& core) { return QueueConsumer(core, schema); }); + .thenApply( + [schema](const detail::QueueConsumerCore& core) { return QueueConsumer(core, schema); }); } private: diff --git a/include/pulsar/st/Schema.h b/include/pulsar/st/Schema.h index 1ad5d649..856bb021 100644 --- a/include/pulsar/st/Schema.h +++ b/include/pulsar/st/Schema.h @@ -67,9 +67,12 @@ using Bytes = std::vector; */ template concept SerDeFor = requires(const S& serde, const T& value, const char* data, std::size_t size) { - { serde.info() } -> std::convertible_to; - { serde.encode(value) } -> std::convertible_to; - { serde.decode(data, size) } -> std::convertible_to; + { serde.info() } + ->std::convertible_to; + { serde.encode(value) } + ->std::convertible_to; + { serde.decode(data, size) } + ->std::convertible_to; }; /** @@ -126,8 +129,9 @@ class Schema { * @param serde the SerDe to adopt; taken by value and stored. */ template - requires(!std::is_same_v, Schema> && SerDeFor, T>) - Schema(SerDe serde) : self_(std::make_shared>>(std::move(serde))) {} + requires(!std::is_same_v, Schema> && SerDeFor, T>) + Schema(SerDe serde) + : self_(std::make_shared>>(std::move(serde))) {} /** * @brief Returns the schema description sent to the broker for compatibility. diff --git a/include/pulsar/st/StreamConsumer.h b/include/pulsar/st/StreamConsumer.h index a0f7c198..8775f890 100644 --- a/include/pulsar/st/StreamConsumer.h +++ b/include/pulsar/st/StreamConsumer.h @@ -19,8 +19,6 @@ #pragma once #include -#include -#include #include #include #include @@ -28,6 +26,8 @@ #include #include #include +#include +#include #include #include @@ -38,7 +38,6 @@ namespace pulsar::st { - /** * Plain-old-data configuration accumulated by `StreamConsumerBuilder`. * @@ -89,9 +88,6 @@ struct StreamConsumerConfig { SchemaInfo schema; }; - - - template class StreamConsumerBuilder; @@ -271,7 +267,6 @@ class StreamConsumerBuilder { return *this; } - /** * Subscribe to all scalable topics in a namespace with live membership — topics * created or removed later are joined/dropped automatically (spec §7.1). @@ -400,7 +395,8 @@ class StreamConsumerBuilder { StreamConsumerConfig config = config_; config.schema = schema.info(); return client_.subscribeStreamAsync(std::move(config)) - .thenApply([schema](const detail::StreamConsumerCore& core) { return StreamConsumer(core, schema); }); + .thenApply( + [schema](const detail::StreamConsumerCore& core) { return StreamConsumer(core, schema); }); } private: diff --git a/include/pulsar/st/Transaction.h b/include/pulsar/st/Transaction.h index 0b69ba5b..a5e7fe01 100644 --- a/include/pulsar/st/Transaction.h +++ b/include/pulsar/st/Transaction.h @@ -43,7 +43,8 @@ class QueueConsumerCore; * terminal states `Committed`, `Aborted`, `Error`, or `TimedOut`. Query the * current state with `Transaction::state()`. */ -enum class TransactionState { +enum class TransactionState +{ Open, ///< Active: messages and acks may still be enlisted; not yet committed or aborted. Committing, ///< Transient: a `commit()`/`commitAsync()` is in progress but not yet durable. Aborting, ///< Transient: an `abort()`/`abortAsync()` is in progress but not yet finalized. diff --git a/include/pulsar/st/detail/Cxx20.h b/include/pulsar/st/detail/Cxx20.h index 1fae2385..698a129d 100644 --- a/include/pulsar/st/detail/Cxx20.h +++ b/include/pulsar/st/detail/Cxx20.h @@ -22,5 +22,6 @@ // client remains C++17 — only this new API requires C++20 (for concepts, // coroutine-awaitable Future, `using enum`, reflection-based schemas, etc.). #if (defined(_MSVC_LANG) ? _MSVC_LANG : __cplusplus) < 202002L -#error "pulsar::st (scalable topics) requires C++20. Build this translation unit with -std=c++20 (or /std:c++20)." +#error \ + "pulsar::st (scalable topics) requires C++20. Build this translation unit with -std=c++20 (or /std:c++20)." #endif diff --git a/vcpkg.json b/vcpkg.json index 721371da..3452492d 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -43,10 +43,6 @@ "name": "protobuf", "version>=": "6.33.4#1" }, - { - "name": "reflectcpp", - "version>=": "0.24.0" - }, { "name": "snappy", "version>=": "1.2.2" From afda5578d93aa4cc4c8461ec32365ed5e80e5bf4 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 23 Jun 2026 14:41:19 -0700 Subject: [PATCH 3/3] Fix CI: give st config-struct fields default member initializers GCC's -Wmissing-field-initializers (-Wextra, and the build is -Werror) fires on a partial designated-initializer such as .deadLetterPolicy({.maxRedeliverCount = 5}) for every omitted member that lacks a default member initializer. clang does not warn, so this was missed locally. Give every optional field in the user-facing policy/ack/DLQ structs an '= std::nullopt' NSDMI so designated-init of any subset is warning-clean. Verified with gcc:13 -Wextra -Werror against all four st examples. Signed-off-by: Matteo Merli --- include/pulsar/st/Consumer.h | 8 ++++---- include/pulsar/st/Policies.h | 26 +++++++++++++------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/include/pulsar/st/Consumer.h b/include/pulsar/st/Consumer.h index 8c346656..a5598126 100644 --- a/include/pulsar/st/Consumer.h +++ b/include/pulsar/st/Consumer.h @@ -54,10 +54,10 @@ enum class SubscriptionInitialPosition struct AckPolicy { /** Time window over which acknowledgments are batched before being sent, in milliseconds; 0 acks * immediately. Unset uses the client default. */ - std::optional groupTime; + std::optional groupTime = std::nullopt; /** Delay before a negatively-acknowledged message is redelivered, in milliseconds. QueueConsumer only. * Unset uses the client default. */ - std::optional negativeAckRedeliveryDelay; + std::optional negativeAckRedeliveryDelay = std::nullopt; }; /** @@ -73,10 +73,10 @@ struct DeadLetterPolicy { * which disables dead-lettering. */ int maxRedeliverCount = 0; /** Name of the dead-letter topic. Unset defaults to "<topic>-<subscription>-DLQ". */ - std::optional deadLetterTopic; + std::optional deadLetterTopic = std::nullopt; /** If set, creates this subscription on the dead-letter topic up front so no messages are missed before a * consumer attaches. Unset creates no initial subscription. */ - std::optional initialSubscriptionName; + std::optional initialSubscriptionName = std::nullopt; }; } // namespace pulsar::st diff --git a/include/pulsar/st/Policies.h b/include/pulsar/st/Policies.h index 29c0d739..dd2b6a0d 100644 --- a/include/pulsar/st/Policies.h +++ b/include/pulsar/st/Policies.h @@ -89,24 +89,24 @@ struct MemorySize { */ struct ConnectionPolicy { /** Number of physical connections opened to each broker. Unset uses the client default. */ - std::optional connectionsPerBroker; + std::optional connectionsPerBroker = std::nullopt; /** Maximum time to wait for a TCP/TLS connection to be established, in milliseconds. Unset uses the * client default. */ - std::optional connectionTimeout; + std::optional connectionTimeout = std::nullopt; /** Maximum time to wait for a broker request (e.g. produce/consume control ops) to complete, in * milliseconds. Unset uses the client default. */ - std::optional operationTimeout; + std::optional operationTimeout = std::nullopt; /** Interval between keep-alive pings sent on an idle connection, in seconds. Unset uses the client * default. */ - std::optional keepAliveInterval; + std::optional keepAliveInterval = std::nullopt; /** Maximum number of concurrent topic-lookup requests in flight. Unset uses the client default. */ - std::optional maxLookupRequests; + std::optional maxLookupRequests = std::nullopt; /** Maximum number of lookup redirects to follow before failing a lookup. Unset uses the client default. */ - std::optional maxLookupRedirects; + std::optional maxLookupRedirects = std::nullopt; /** Time an idle pooled connection may stay open before being closed, in milliseconds. Unset uses the * client default. */ - std::optional maxConnectionIdleTime; + std::optional maxConnectionIdleTime = std::nullopt; }; /** @@ -118,10 +118,10 @@ struct ConnectionPolicy { */ struct BackoffPolicy { /** Delay before the first reconnection attempt, in milliseconds. Unset uses the client default. */ - std::optional initialBackoff; + std::optional initialBackoff = std::nullopt; /** Upper bound on the backoff delay as it grows across retries, in milliseconds. Unset uses the client * default. */ - std::optional maxBackoff; + std::optional maxBackoff = std::nullopt; }; /** @@ -136,13 +136,13 @@ struct TlsPolicy { bool enabled = false; /** Path to the PEM file of trusted CA certificates used to verify the broker. Unset uses the system trust * store. */ - std::optional trustCertsFilePath; + std::optional trustCertsFilePath = std::nullopt; /** Path to the client certificate PEM file, for mutual TLS. Unset disables client-certificate * authentication. */ - std::optional certificateFilePath; + std::optional certificateFilePath = std::nullopt; /** Path to the client private key PEM file, for mutual TLS. Unset disables client-certificate * authentication. */ - std::optional privateKeyFilePath; + std::optional privateKeyFilePath = std::nullopt; /** Whether to accept the broker's certificate without validating it against the trust store. Defaults to * false (validation enforced). */ bool allowInsecureConnection = false; @@ -160,7 +160,7 @@ struct TlsPolicy { struct TransactionPolicy { /** Default lifetime of a transaction before it is automatically aborted, in milliseconds. Unset uses the * client default. */ - std::optional timeout; + std::optional timeout = std::nullopt; }; } // namespace pulsar::st