From 0c24aa84df5126f4e4b81c10f97c3346127dc357 Mon Sep 17 00:00:00 2001 From: Vikentiy Fesunov Date: Thu, 20 Aug 2026 14:37:17 +0200 Subject: [PATCH] Promote host tag to a resource Extract old magic host override from tags, if present, and set it as a resource instead. --- .../dogstatsd/http/DirectHttpClient.java | 78 ++++++- .../dogstatsd/http/serializer/Metric.java | 10 +- .../dogstatsd/http/DirectHttpClientTest.java | 207 ++++++++++++++++++ 3 files changed, 284 insertions(+), 11 deletions(-) create mode 100644 dogstatsd-http/core/src/test/java/com/datadoghq/dogstatsd/http/DirectHttpClientTest.java diff --git a/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/DirectHttpClient.java b/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/DirectHttpClient.java index 7bb83bc5..886b68d7 100644 --- a/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/DirectHttpClient.java +++ b/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/DirectHttpClient.java @@ -8,16 +8,22 @@ package com.datadoghq.dogstatsd.http; import com.datadoghq.dogstatsd.Sketch; +import com.datadoghq.dogstatsd.http.serializer.Metric; import com.datadoghq.dogstatsd.http.serializer.PayloadBuilder; import com.datadoghq.dogstatsd.http.serializer.PayloadConsumer; import java.net.URI; import java.nio.BufferOverflowException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; /** * Simple Dogstatsd HTTP client for sending pre-aggregated metrics. * + *

A {@code host:} tag is not sent as a tag: it is removed from the tags and submitted as the + * host resource of the timeseries. + * *

Not thread safe. * *

Caveat: if the forwarder throws {@code InterruptedException}, the payload in progress is lost. @@ -30,6 +36,8 @@ public class DirectHttpClient { private final Sketch sketchBuffer = new Sketch(); private final String prefix; private static final int defaultInterval = 10; + private static final String hostTagPrefix = "host:"; + private static final String hostResourceType = "host"; /** * Creates a builder for a client sending its payloads through the given forwarder. @@ -127,13 +135,13 @@ public DirectHttpClient build() { * @param name the metric name, to which the client prefix is prepended. * @param value the gauge value. * @param ts the timestamp of the point in seconds since Unix epoch. - * @param tags the tags to attach to the point. + * @param tags the tags to attach to the point. A {@code host:} tag is not attached as a tag: + * the first one is submitted as the host resource of the timeseries, and any further {@code + * host:} tags are dropped. * @throws BufferOverflowException if the encoded metric exceeds the maximum payload size. */ public void gauge(String name, double value, long ts, List tags) { - seriesBuilder - .gauge(prefixed(name)) - .setTags(tags) + withTagsAndHost(seriesBuilder.gauge(prefixed(name)), tags) .setInterval(defaultInterval) .addPoint(ts, value) .close(); @@ -148,13 +156,13 @@ public void gauge(String name, double value, long ts, List tags) { * @param name the metric name, to which the client prefix is prepended. * @param value the count accumulated over the interval starting at {@code ts}. * @param ts the timestamp of the point in seconds since Unix epoch. - * @param tags the tags to attach to the point. + * @param tags the tags to attach to the point. A {@code host:} tag is not attached as a tag: + * the first one is submitted as the host resource of the timeseries, and any further {@code + * host:} tags are dropped. * @throws BufferOverflowException if the encoded metric exceeds the maximum payload size. */ public void count(String name, double value, long ts, List tags) { - seriesBuilder - .rate(prefixed(name)) - .setTags(tags) + withTagsAndHost(seriesBuilder.rate(prefixed(name)), tags) .setInterval(defaultInterval) .addPoint(ts, value / defaultInterval) .close(); @@ -167,7 +175,9 @@ public void count(String name, double value, long ts, List tags) { * @param values the observations to summarize. * @param sampleRate the sampling rate used to collect {@code values}, in {@code (0, 1]}. * @param ts the timestamp of the point in seconds since Unix epoch. - * @param tags the tags to attach to the point. + * @param tags the tags to attach to the point. A {@code host:} tag is not attached as a tag: + * the first one is submitted as the host resource of the timeseries, and any further {@code + * host:} tags are dropped. * @throws IllegalArgumentException if {@code sampleRate} is {@code NaN}, not positive, or * greater than 1. * @throws BufferOverflowException if the encoded metric exceeds the maximum payload size. @@ -175,13 +185,61 @@ public void count(String name, double value, long ts, List tags) { public void distribution( String name, double[] values, double sampleRate, long ts, List tags) { sketchBuffer.build(values, sampleRate); - sketchesBuilder.sketch(prefixed(name)).setTags(tags).addPoint(ts, sketchBuffer).close(); + withTagsAndHost(sketchesBuilder.sketch(prefixed(name)), tags) + .addPoint(ts, sketchBuffer) + .close(); } private String prefixed(final String name) { return prefix.isEmpty() ? name : prefix + name; } + /** Applies the tags to the metric, extracting the host tag into the host resource. */ + private static > T withTagsAndHost( + final T metric, final List tags) { + final String host = hostTag(tags); + return metric.setTags(host == null ? tags : withoutHostTags(tags)) + .setResources(hostResource(host)); + } + + /** Returns the value of the first host tag, or null if there is none. */ + static String hostTag(final List tags) { + if (tags == null) { + return null; + } + for (int i = 0; i < tags.size(); i++) { + final String tag = tags.get(i); + if (tag.startsWith(hostTagPrefix)) { + return tag.substring(hostTagPrefix.length()); + } + } + return null; + } + + /** Returns the tags with every host tag removed, or the tags themselves if there was none. */ + static List withoutHostTags(final List tags) { + if (tags == null) { + return null; + } + ArrayList rest = null; + for (int i = 0; i < tags.size(); i++) { + final String tag = tags.get(i); + if (tag.startsWith(hostTagPrefix)) { + if (rest == null) { + rest = new ArrayList<>(tags.subList(0, i)); + } + } else if (rest != null) { + rest.add(tag); + } + } + return rest == null ? tags : rest; + } + + /** Returns the host resource pair, or null if there is no host. */ + static List hostResource(final String host) { + return host == null ? null : Arrays.asList(hostResourceType, host); + } + /** * Completes any in-progress payloads and submits them to the forwarder. * diff --git a/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/serializer/Metric.java b/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/serializer/Metric.java index ac28a0e1..756b931f 100644 --- a/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/serializer/Metric.java +++ b/dogstatsd-http/core/src/main/java/com/datadoghq/dogstatsd/http/serializer/Metric.java @@ -9,7 +9,15 @@ import java.util.List; -abstract class Metric> { +/** + * Common settings shared by all timeseries builders. + * + *

Instances are obtained from {@link PayloadBuilder}; this class cannot be subclassed outside of + * this package. + * + * @param the concrete builder type returned by the setters, for chaining. + */ +public abstract class Metric> { final PayloadBuilder pb; final long type; diff --git a/dogstatsd-http/core/src/test/java/com/datadoghq/dogstatsd/http/DirectHttpClientTest.java b/dogstatsd-http/core/src/test/java/com/datadoghq/dogstatsd/http/DirectHttpClientTest.java new file mode 100644 index 00000000..ece4b741 --- /dev/null +++ b/dogstatsd-http/core/src/test/java/com/datadoghq/dogstatsd/http/DirectHttpClientTest.java @@ -0,0 +1,207 @@ +/* Unless explicitly stated otherwise all files in this repository are + * licensed under the Apache 2.0 License. + * + * This product includes software developed at Datadog + * (https://www.datadoghq.com/) Copyright 2026 Datadog, Inc. + */ + +package com.datadoghq.dogstatsd.http; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; + +import com.datadoghq.dogstatsd.Sketch; +import com.datadoghq.dogstatsd.http.serializer.PayloadBuilder; +import com.datadoghq.dogstatsd.http.serializer.PayloadConsumer; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.Test; + +public class DirectHttpClientTest { + private static final URI seriesUri = URI.create("series"); + private static final URI sketchesUri = URI.create("sketches"); + + static class TestForwarder implements DirectHttpClient.Forwarder { + final ArrayList uris = new ArrayList<>(); + final ArrayList payloads = new ArrayList<>(); + + @Override + public void send(URI uri, byte[] payload) { + uris.add(uri); + payloads.add(payload); + } + } + + private static PayloadBuilder builderInto(final List payloads) { + return new PayloadBuilder( + new PayloadConsumer() { + @Override + public void handle(byte[] p) { + payloads.add(p); + } + }); + } + + /** Asserts the client sent exactly one payload to uri, matching the only expected payload. */ + private static void assertSent(List expected, TestForwarder fwd, URI uri, String what) { + assertEquals("expected payloads", 1, expected.size()); + assertEquals("payloads sent", Collections.singletonList(uri), fwd.uris); + assertArrayEquals(what, expected.get(0), fwd.payloads.get(0)); + } + + @Test + public void gaugeSendsHostTagAsResource() { + TestForwarder fwd = new TestForwarder(); + DirectHttpClient client = DirectHttpClient.builder(fwd).prefix("pfx").build(); + client.gauge("metric", 1.5, 100, Arrays.asList("a:b", "host:h1", "host:h2")); + client.flush(); + + ArrayList expected = new ArrayList<>(); + PayloadBuilder b = builderInto(expected); + b.gauge("pfx.metric") + .setTags(Collections.singletonList("a:b")) + .setResources(Arrays.asList("host", "h1")) + .setInterval(10) + .addPoint(100, 1.5) + .close(); + b.close(); + + assertSent(expected, fwd, seriesUri, "gauge"); + } + + @Test + public void countSendsHostTagAsResource() { + TestForwarder fwd = new TestForwarder(); + DirectHttpClient client = DirectHttpClient.builder(fwd).build(); + client.count("metric", 20, 100, Arrays.asList("host:h1", "a:b")); + client.flush(); + + ArrayList expected = new ArrayList<>(); + PayloadBuilder b = builderInto(expected); + b.rate("metric") + .setTags(Collections.singletonList("a:b")) + .setResources(Arrays.asList("host", "h1")) + .setInterval(10) + .addPoint(100, 2) + .close(); + b.close(); + + assertSent(expected, fwd, seriesUri, "count"); + } + + @Test + public void distributionSendsHostTagAsResource() { + double[] values = new double[] {1, 2, 2}; + + TestForwarder fwd = new TestForwarder(); + DirectHttpClient client = DirectHttpClient.builder(fwd).build(); + client.distribution("metric", values, 1.0, 100, Arrays.asList("host:h1", "a:b")); + client.flush(); + + Sketch sketch = new Sketch(); + sketch.build(values, 1.0); + + ArrayList expected = new ArrayList<>(); + PayloadBuilder b = builderInto(expected); + b.sketch("metric") + .setTags(Collections.singletonList("a:b")) + .setResources(Arrays.asList("host", "h1")) + .addPoint(100, sketch) + .close(); + b.close(); + + assertSent(expected, fwd, sketchesUri, "distribution"); + } + + @Test + public void noHostTagLeavesResourcesUnset() { + TestForwarder fwd = new TestForwarder(); + DirectHttpClient client = DirectHttpClient.builder(fwd).build(); + client.gauge("metric", 1.5, 100, Arrays.asList("a:b", "hostname:h1")); + client.flush(); + + ArrayList expected = new ArrayList<>(); + PayloadBuilder b = builderInto(expected); + b.gauge("metric") + .setTags(Arrays.asList("a:b", "hostname:h1")) + .setInterval(10) + .addPoint(100, 1.5) + .close(); + b.close(); + + assertSent(expected, fwd, seriesUri, "gauge without host tag"); + } + + @Test + public void emptyHostTagSendsEmptyHostResource() { + TestForwarder fwd = new TestForwarder(); + DirectHttpClient client = DirectHttpClient.builder(fwd).build(); + client.gauge("metric", 1.5, 100, Arrays.asList("a:b", "host:")); + client.flush(); + + ArrayList expected = new ArrayList<>(); + PayloadBuilder b = builderInto(expected); + b.gauge("metric") + .setTags(Collections.singletonList("a:b")) + .setResources(Arrays.asList("host", "")) + .setInterval(10) + .addPoint(100, 1.5) + .close(); + b.close(); + + assertSent(expected, fwd, seriesUri, "gauge with empty host tag"); + + // An empty host resource must still be encoded, unlike a metric with no resources at all. + ArrayList noResources = new ArrayList<>(); + PayloadBuilder nb = builderInto(noResources); + nb.gauge("metric") + .setTags(Collections.singletonList("a:b")) + .setInterval(10) + .addPoint(100, 1.5) + .close(); + nb.close(); + + assertFalse( + "empty host resource encodes the same as no resources", + Arrays.equals(noResources.get(0), fwd.payloads.get(0))); + } + + @Test + public void hostTagPicksFirstValue() { + assertNull(DirectHttpClient.hostTag(null)); + assertNull(DirectHttpClient.hostTag(Collections.emptyList())); + assertNull(DirectHttpClient.hostTag(Arrays.asList("a:b", "hostname:h1", "hos:t"))); + assertEquals("h1", DirectHttpClient.hostTag(Collections.singletonList("host:h1"))); + assertEquals("h1", DirectHttpClient.hostTag(Arrays.asList("a:b", "host:h1", "host:h2"))); + assertEquals("", DirectHttpClient.hostTag(Collections.singletonList("host:"))); + } + + @Test + public void withoutHostTagsRemovesEveryHostTag() { + assertNull(DirectHttpClient.withoutHostTags(null)); + + List noHost = Arrays.asList("a:b", "hostname:h1"); + assertSame(noHost, DirectHttpClient.withoutHostTags(noHost)); + + assertEquals( + Collections.emptyList(), + DirectHttpClient.withoutHostTags(Arrays.asList("host:h1", "host:"))); + assertEquals( + Arrays.asList("a:b", "c:d"), + DirectHttpClient.withoutHostTags( + Arrays.asList("a:b", "host:h1", "c:d", "host:h2"))); + } + + @Test + public void hostResourceIsATypeNamePair() { + assertNull(DirectHttpClient.hostResource(null)); + assertEquals(Arrays.asList("host", "h1"), DirectHttpClient.hostResource("h1")); + assertEquals(Arrays.asList("host", ""), DirectHttpClient.hostResource("")); + } +}