diff --git a/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/DirectHttpClient.java b/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/DirectHttpClient.java index 886b68d7..00266a8f 100644 --- a/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/DirectHttpClient.java +++ b/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/DirectHttpClient.java @@ -11,6 +11,7 @@ import com.datadoghq.dogstatsd.http.serializer.Metric; import com.datadoghq.dogstatsd.http.serializer.PayloadBuilder; import com.datadoghq.dogstatsd.http.serializer.PayloadConsumer; +import com.datadoghq.dogstatsd.http.serializer.TagsCardinality; import java.net.URI; import java.nio.BufferOverflowException; import java.util.ArrayList; @@ -21,8 +22,17 @@ /** * Simple Dogstatsd HTTP client for sending pre-aggregated metrics. * - *

A {@code host:} tag is not sent as a tag: it is removed from the tags and submitted as the - * host resource of the timeseries. + *

Two tags are given special treatment: their value is submitted as a property of the + * timeseries. + * + *

* *

Not thread safe. * @@ -38,6 +48,7 @@ public class DirectHttpClient { private static final int defaultInterval = 10; private static final String hostTagPrefix = "host:"; private static final String hostResourceType = "host"; + private static final String cardinalityTagPrefix = "dd.internal.card:"; /** * Creates a builder for a client sending its payloads through the given forwarder. @@ -135,13 +146,12 @@ public DirectHttpClient build() { * @param name the metric name, to which the client prefix is prepended. * @param value the gauge value. * @param ts the timestamp of the point in seconds since Unix epoch. - * @param tags the tags to attach to the point. A {@code host:} tag is not attached as a tag: - * the first one is submitted as the host resource of the timeseries, and any further {@code - * host:} tags are dropped. + * @param tags the tags to attach to the point, see {@link DirectHttpClient} for the handling of + * special values. * @throws BufferOverflowException if the encoded metric exceeds the maximum payload size. */ public void gauge(String name, double value, long ts, List tags) { - withTagsAndHost(seriesBuilder.gauge(prefixed(name)), tags) + withTagsHostAndCardinality(seriesBuilder.gauge(prefixed(name)), tags) .setInterval(defaultInterval) .addPoint(ts, value) .close(); @@ -156,13 +166,12 @@ public void gauge(String name, double value, long ts, List tags) { * @param name the metric name, to which the client prefix is prepended. * @param value the count accumulated over the interval starting at {@code ts}. * @param ts the timestamp of the point in seconds since Unix epoch. - * @param tags the tags to attach to the point. A {@code host:} tag is not attached as a tag: - * the first one is submitted as the host resource of the timeseries, and any further {@code - * host:} tags are dropped. + * @param tags the tags to attach to the point, see {@link DirectHttpClient} for the handling of + * special values. * @throws BufferOverflowException if the encoded metric exceeds the maximum payload size. */ public void count(String name, double value, long ts, List tags) { - withTagsAndHost(seriesBuilder.rate(prefixed(name)), tags) + withTagsHostAndCardinality(seriesBuilder.rate(prefixed(name)), tags) .setInterval(defaultInterval) .addPoint(ts, value / defaultInterval) .close(); @@ -175,9 +184,8 @@ public void count(String name, double value, long ts, List tags) { * @param values the observations to summarize. * @param sampleRate the sampling rate used to collect {@code values}, in {@code (0, 1]}. * @param ts the timestamp of the point in seconds since Unix epoch. - * @param tags the tags to attach to the point. A {@code host:} tag is not attached as a tag: - * the first one is submitted as the host resource of the timeseries, and any further {@code - * host:} tags are dropped. + * @param tags the tags to attach to the point, see {@link DirectHttpClient} for the handling of + * special values. * @throws IllegalArgumentException if {@code sampleRate} is {@code NaN}, not positive, or * greater than 1. * @throws BufferOverflowException if the encoded metric exceeds the maximum payload size. @@ -185,7 +193,7 @@ public void count(String name, double value, long ts, List tags) { public void distribution( String name, double[] values, double sampleRate, long ts, List tags) { sketchBuffer.build(values, sampleRate); - withTagsAndHost(sketchesBuilder.sketch(prefixed(name)), tags) + withTagsHostAndCardinality(sketchesBuilder.sketch(prefixed(name)), tags) .addPoint(ts, sketchBuffer) .close(); } @@ -194,28 +202,64 @@ private String prefixed(final String name) { return prefix.isEmpty() ? name : prefix + name; } - /** Applies the tags to the metric, extracting the host tag into the host resource. */ - private static > T withTagsAndHost( + /** + * Applies the tags to the metric, extracting the host tag into the host resource and the + * cardinality tag into the tags cardinality. The cardinality tag itself is kept in the tags. + */ + private static > T withTagsHostAndCardinality( final T metric, final List tags) { - final String host = hostTag(tags); - return metric.setTags(host == null ? tags : withoutHostTags(tags)) - .setResources(hostResource(host)); + return metric.setTags(withoutHostTags(tags)) + .setResources(hostResource(hostTag(tags))) + .setTagsCardinality(cardinality(cardinalityTag(tags))); } /** Returns the value of the first host tag, or null if there is none. */ static String hostTag(final List tags) { + return tagValue(tags, hostTagPrefix); + } + + /** Returns the value of the first cardinality tag, or null if there is none. */ + static String cardinalityTag(final List tags) { + return tagValue(tags, cardinalityTagPrefix); + } + + /** Returns the value of the first tag with the given prefix, or null if there is none. */ + private static String tagValue(final List tags, final String prefix) { if (tags == null) { return null; } for (int i = 0; i < tags.size(); i++) { final String tag = tags.get(i); - if (tag.startsWith(hostTagPrefix)) { - return tag.substring(hostTagPrefix.length()); + if (tag.startsWith(prefix)) { + return tag.substring(prefix.length()); } } return null; } + /** + * Returns the cardinality constant matching the value of a cardinality tag. Values the agent + * does not recognize, including null, ask for the cardinality the agent is configured to use. + */ + static TagsCardinality cardinality(final String value) { + if (value == null) { + return TagsCardinality.DEFAULT; + } + if (value.equalsIgnoreCase("none")) { + return TagsCardinality.NONE; + } + if (value.equalsIgnoreCase("low")) { + return TagsCardinality.LOW; + } + if (value.equalsIgnoreCase("orchestrator") || value.equalsIgnoreCase("orch")) { + return TagsCardinality.ORCHESTRATOR; + } + if (value.equalsIgnoreCase("high")) { + return TagsCardinality.HIGH; + } + return TagsCardinality.DEFAULT; + } + /** Returns the tags with every host tag removed, or the tags themselves if there was none. */ static List withoutHostTags(final List tags) { if (tags == null) { diff --git a/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/serializer/Metric.java b/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/serializer/Metric.java index 756b931f..2fbab00f 100644 --- a/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/serializer/Metric.java +++ b/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/serializer/Metric.java @@ -25,6 +25,7 @@ public abstract class Metric> { List tags = null; List resources = null; int interval = 0; + TagsCardinality cardinality = TagsCardinality.DEFAULT; Origin origin = Origin.dogstatsd; Metric(PayloadBuilder pb, int type, String name) { @@ -71,6 +72,19 @@ public T setInterval(int interval) { return self(); } + /** + * Set the cardinality of the origin tags the agent attaches to this metric. + * + *

Ignored by the intake, which does not attach origin tags. + * + * @param cardinality The cardinality to request, or null for {@link TagsCardinality#DEFAULT}. + * @return This. + */ + public T setTagsCardinality(TagsCardinality cardinality) { + this.cardinality = cardinality == null ? TagsCardinality.DEFAULT : cardinality; + return self(); + } + abstract T self(); abstract void encodeValues(ValueType valueType); @@ -78,7 +92,7 @@ public T setInterval(int interval) { void encodeIndependentFields() { ColumnarBuffer r = pb.currentRecord(); ValueType valueType = PointKind.of(pb.values).toValueType(); - r.putUint64(Column.types, type | valueType.flag()); + r.putUint64(Column.types, type | valueType.flag() | cardinality.flag()); r.putUint64(Column.intervals, interval); r.putSint64(Column.sourceTypeNameRefs, 0); encodeValues(valueType); diff --git a/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/serializer/TagsCardinality.java b/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/serializer/TagsCardinality.java new file mode 100644 index 00000000..b1648f08 --- /dev/null +++ b/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/serializer/TagsCardinality.java @@ -0,0 +1,42 @@ +/* Unless explicitly stated otherwise all files in this repository are + * licensed under the Apache 2.0 License. + * + * This product includes software developed at Datadog + * (https://www.datadoghq.com/) Copyright 2026 Datadog, Inc. + */ + +package com.datadoghq.dogstatsd.http.serializer; + +/** + * Cardinality of the origin tags the agent attaches to a metric. + * + *

See the tags + * cardinality documentation. + */ +public enum TagsCardinality { + /** Requests the cardinality the agent is configured to use for dogstatsd metrics. */ + DEFAULT(0x0000), + + /** Requests no origin tags at all. */ + NONE(0x1000), + + /** Requests low cardinality origin tags. */ + LOW(0x2000), + + /** Requests orchestrator cardinality origin tags. */ + ORCHESTRATOR(0x3000), + + /** Requests high cardinality origin tags. */ + HIGH(0x4000); + + private final int flag; + + TagsCardinality(int flag) { + this.flag = flag; + } + + int flag() { + return flag; + } +} diff --git a/dogstatsd-http/core/src/test/java/com/datadoghq/dogstatsd/http/DirectHttpClientTest.java b/dogstatsd-http/core/src/test/java/com/datadoghq/dogstatsd/http/DirectHttpClientTest.java index ece4b741..1cc699e7 100644 --- a/dogstatsd-http/core/src/test/java/com/datadoghq/dogstatsd/http/DirectHttpClientTest.java +++ b/dogstatsd-http/core/src/test/java/com/datadoghq/dogstatsd/http/DirectHttpClientTest.java @@ -16,6 +16,7 @@ import com.datadoghq.dogstatsd.Sketch; import com.datadoghq.dogstatsd.http.serializer.PayloadBuilder; import com.datadoghq.dogstatsd.http.serializer.PayloadConsumer; +import com.datadoghq.dogstatsd.http.serializer.TagsCardinality; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; @@ -196,6 +197,10 @@ public void withoutHostTagsRemovesEveryHostTag() { Arrays.asList("a:b", "c:d"), DirectHttpClient.withoutHostTags( Arrays.asList("a:b", "host:h1", "c:d", "host:h2"))); + + // The cardinality tag is sent to the agent as a tag as well. + List card = Arrays.asList("a:b", "dd.internal.card:low"); + assertSame(card, DirectHttpClient.withoutHostTags(card)); } @Test @@ -204,4 +209,128 @@ public void hostResourceIsATypeNamePair() { assertEquals(Arrays.asList("host", "h1"), DirectHttpClient.hostResource("h1")); assertEquals(Arrays.asList("host", ""), DirectHttpClient.hostResource("")); } + + @Test + public void noCardinalityTagLeavesCardinalityDefault() { + TestForwarder fwd = new TestForwarder(); + DirectHttpClient client = DirectHttpClient.builder(fwd).build(); + client.gauge("metric", 1.5, 100, Collections.singletonList("a:b")); + client.flush(); + + ArrayList expected = new ArrayList<>(); + PayloadBuilder b = builderInto(expected); + b.gauge("metric") + .setTags(Collections.singletonList("a:b")) + .setTagsCardinality(TagsCardinality.DEFAULT) + .setInterval(10) + .addPoint(100, 1.5) + .close(); + b.close(); + + assertSent(expected, fwd, seriesUri, "gauge without cardinality"); + } + + @Test + public void cardinalityTagIsSentAsCardinality() { + TestForwarder fwd = new TestForwarder(); + DirectHttpClient client = DirectHttpClient.builder(fwd).build(); + client.gauge( + "gauge", + 1.5, + 100, + Arrays.asList("a:b", "dd.internal.card:high", "host:h1", "dd.internal.card:low")); + client.count("count", 20, 100, Arrays.asList("dd.internal.card:none", "a:b")); + client.flush(); + + ArrayList expected = new ArrayList<>(); + PayloadBuilder b = builderInto(expected); + b.gauge("gauge") + .setTags(Arrays.asList("a:b", "dd.internal.card:high", "dd.internal.card:low")) + .setResources(Arrays.asList("host", "h1")) + .setTagsCardinality(TagsCardinality.HIGH) + .setInterval(10) + .addPoint(100, 1.5) + .close(); + b.rate("count") + .setTags(Arrays.asList("dd.internal.card:none", "a:b")) + .setTagsCardinality(TagsCardinality.NONE) + .setInterval(10) + .addPoint(100, 2) + .close(); + b.close(); + + assertSent(expected, fwd, seriesUri, "gauge and count with cardinality tag"); + } + + @Test + public void distributionSendsCardinalityTagAsCardinality() { + double[] values = new double[] {1, 2, 2}; + + TestForwarder fwd = new TestForwarder(); + DirectHttpClient client = DirectHttpClient.builder(fwd).build(); + client.distribution( + "metric", values, 1.0, 100, Arrays.asList("dd.internal.card:orch", "a:b")); + client.flush(); + + Sketch sketch = new Sketch(); + sketch.build(values, 1.0); + + ArrayList expected = new ArrayList<>(); + PayloadBuilder b = builderInto(expected); + b.sketch("metric") + .setTags(Arrays.asList("dd.internal.card:orch", "a:b")) + .setTagsCardinality(TagsCardinality.ORCHESTRATOR) + .addPoint(100, sketch) + .close(); + b.close(); + + assertSent(expected, fwd, sketchesUri, "distribution with cardinality tag"); + } + + @Test + public void unknownCardinalityTagLeavesCardinalityDefault() { + TestForwarder fwd = new TestForwarder(); + DirectHttpClient client = DirectHttpClient.builder(fwd).build(); + client.gauge("metric", 1.5, 100, Arrays.asList("a:b", "dd.internal.card:bogus")); + client.flush(); + + ArrayList expected = new ArrayList<>(); + PayloadBuilder b = builderInto(expected); + b.gauge("metric") + .setTags(Arrays.asList("a:b", "dd.internal.card:bogus")) + .setInterval(10) + .addPoint(100, 1.5) + .close(); + b.close(); + + assertSent(expected, fwd, seriesUri, "gauge with unknown cardinality tag"); + } + + @Test + public void cardinalityTagPicksFirstValue() { + assertNull(DirectHttpClient.cardinalityTag(null)); + assertNull(DirectHttpClient.cardinalityTag(Collections.emptyList())); + assertNull(DirectHttpClient.cardinalityTag(Arrays.asList("a:b", "dd.internal.cardx:low"))); + assertEquals( + "low", + DirectHttpClient.cardinalityTag( + Arrays.asList("a:b", "dd.internal.card:low", "dd.internal.card:high"))); + assertEquals( + "", + DirectHttpClient.cardinalityTag(Collections.singletonList("dd.internal.card:"))); + } + + @Test + public void cardinalityParsesTheValuesTheAgentAccepts() { + assertEquals(TagsCardinality.DEFAULT, DirectHttpClient.cardinality(null)); + assertEquals(TagsCardinality.DEFAULT, DirectHttpClient.cardinality("")); + assertEquals(TagsCardinality.DEFAULT, DirectHttpClient.cardinality("bogus")); + assertEquals(TagsCardinality.NONE, DirectHttpClient.cardinality("none")); + assertEquals(TagsCardinality.LOW, DirectHttpClient.cardinality("low")); + assertEquals(TagsCardinality.ORCHESTRATOR, DirectHttpClient.cardinality("orch")); + assertEquals(TagsCardinality.ORCHESTRATOR, DirectHttpClient.cardinality("orchestrator")); + assertEquals(TagsCardinality.HIGH, DirectHttpClient.cardinality("high")); + // The agent parses the value case-insensitively. + assertEquals(TagsCardinality.HIGH, DirectHttpClient.cardinality("HIGH")); + } } diff --git a/dogstatsd-http/core/src/test/java/com/datadoghq/dogstatsd/http/serializer/PayloadBuilderTest.java b/dogstatsd-http/core/src/test/java/com/datadoghq/dogstatsd/http/serializer/PayloadBuilderTest.java index 6f868fe6..a4929f10 100644 --- a/dogstatsd-http/core/src/test/java/com/datadoghq/dogstatsd/http/serializer/PayloadBuilderTest.java +++ b/dogstatsd-http/core/src/test/java/com/datadoghq/dogstatsd/http/serializer/PayloadBuilderTest.java @@ -10,11 +10,13 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import com.datadoghq.dogstatsd.Sketch; import java.nio.BufferOverflowException; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import org.junit.Test; import org.junit.function.ThrowingRunnable; @@ -439,4 +441,73 @@ public void run() { TestUtil.assertPayload(payloads2.get(i), payloads1.get(i)); } } + + @Test + public void tagsCardinality() { + final ArrayList payloads = new ArrayList<>(); + PayloadBuilder b = + new PayloadBuilder( + new PayloadConsumer() { + @Override + public void handle(byte[] p) { + payloads.add(p); + } + }); + + Sketch sketch = new Sketch(); + sketch.build(new long[] {1, 2, 2}, 1.0); + + b.gauge("default").addPoint(100, 1).close(); + b.gauge("none").setTagsCardinality(TagsCardinality.NONE).addPoint(100, 1).close(); + b.gauge("low").setTagsCardinality(TagsCardinality.LOW).addPoint(100, 1).close(); + b.count("orch").setTagsCardinality(TagsCardinality.ORCHESTRATOR).addPoint(100, 1).close(); + b.sketch("high").setTagsCardinality(TagsCardinality.HIGH).addPoint(100, sketch).close(); + b.close(); + + assertEquals(1, payloads.size()); + // The cardinality occupies the nibble above the metric flags, metric type and value type + // keep the low nibbles. + assertEquals( + Arrays.asList( + Long.valueOf(0x0013), + Long.valueOf(0x1013), + Long.valueOf(0x2013), + Long.valueOf(0x3011), + Long.valueOf(0x4014)), + readUint64Column(payloads.get(0), 10)); + } + + /** Decodes the packed uint64 column stored in the MetricData field with the given id. */ + private static List readUint64Column(byte[] payload, int fieldId) { + TestUtil.Varint var = new TestUtil.Varint(); + var.read(payload, 0); // MetricData field header + int idx = var.len; + var.read(payload, idx); // MetricData length + int end = idx + var.len + var.val; + idx += var.len; + + while (idx < end) { + var.read(payload, idx); + final int id = var.val >> 3; + idx += var.len; + var.read(payload, idx); + final int len = var.val; + idx += var.len; + if (id == fieldId) { + ArrayList vals = new ArrayList<>(); + for (int i = idx; i < idx + len; i += var.len) { + var.read(payload, i); + vals.add(Long.valueOf(var.val)); + } + return vals; + } + idx += len; + } + + fail( + String.format( + "field %d not found in payload:%n%s", + fieldId, TestUtil.protodump(payload))); + return null; + } }