diff --git a/.github/workflows/parparvm-tests-windows.yml b/.github/workflows/parparvm-tests-windows.yml index a96f376497f..62b427fd1bd 100644 --- a/.github/workflows/parparvm-tests-windows.yml +++ b/.github/workflows/parparvm-tests-windows.yml @@ -243,6 +243,17 @@ jobs: # time and cannot turn a failing test green. $ok = $false foreach ($delay in 0, 30, 90, 180, 300) { + # Maven records a download that failed as a *.lastUpdated marker and then + # answers later resolutions from it instead of the network -- the "(absent)" + # in "Could not transfer artifact ... (absent)". Left in place, every retry + # below replays the first 403 offline and cannot succeed: one Central refusal + # of org.junit:junit-bom took all five attempts and the job. And because + # setup-java's cache: 'maven' restores ~/.m2 between runs, a marker can also + # arrive from an earlier run, so this clears them before EVERY attempt, not + # just the retries. A marker records only a failure, never an artifact, so + # deleting one costs at most a repeated question to Central. + Get-ChildItem -Path "$env:USERPROFILE\.m2\repository" -Recurse -File -Filter '*.lastUpdated' -ErrorAction SilentlyContinue | + Remove-Item -Force -ErrorAction SilentlyContinue if ($delay -gt 0) { Write-Host "mvn failed; retrying in $delay s..." Start-Sleep -Seconds $delay @@ -255,9 +266,12 @@ jobs: # exists for. A blanket loop lets an intermittent product regression pass on # attempt two and turns a blocking gate green -- which is the opposite of what # a gate is for, and worse than the flake it was hiding. - $resolutionFailure = 'status: (403|429|50[0-9])|Could not transfer artifact|Failed to read artifact descriptor|Unresolveable build extension|Non-resolvable import POM|Could not resolve dependencies' + $resolutionFailure = 'status: (403|429|50[0-9])|Could not transfer artifact|Failed to read artifact descriptor|Unresolveable build extension|Non-resolvable import POM|Could not resolve dependencies|UnknownHostException|Could not resolve host|Temporary failure in name resolution' $ok = $false foreach ($delay in 0, 30, 90, 180, 300) { + # Clear Maven's cached resolution failures first; see the first retry loop above. + Get-ChildItem -Path "$env:USERPROFILE\.m2\repository" -Recurse -File -Filter '*.lastUpdated' -ErrorAction SilentlyContinue | + Remove-Item -Force -ErrorAction SilentlyContinue if ($delay -gt 0) { Write-Host "mvn failed on a dependency-resolution error; retrying in $delay s..." Start-Sleep -Seconds $delay @@ -371,6 +385,9 @@ jobs: run: | $ok = $false foreach ($delay in 0, 30, 90, 180, 300) { + # Clear Maven's cached resolution failures first; see the first retry loop above. + Get-ChildItem -Path "$env:USERPROFILE\.m2\repository" -Recurse -File -Filter '*.lastUpdated' -ErrorAction SilentlyContinue | + Remove-Item -Force -ErrorAction SilentlyContinue if ($delay -gt 0) { Write-Host "mvn failed; retrying in $delay s..." Start-Sleep -Seconds $delay @@ -404,6 +421,9 @@ jobs: # Same retry as the port build above: this reaches Maven Central too. $ok = $false foreach ($delay in 0, 30, 90, 180, 300) { + # Clear Maven's cached resolution failures first; see the first retry loop above. + Get-ChildItem -Path "$env:USERPROFILE\.m2\repository" -Recurse -File -Filter '*.lastUpdated' -ErrorAction SilentlyContinue | + Remove-Item -Force -ErrorAction SilentlyContinue if ($delay -gt 0) { Write-Host "mvn failed; retrying in $delay s..." Start-Sleep -Seconds $delay @@ -481,6 +501,9 @@ jobs: # blocking gate green. $ok = $false foreach ($delay in 0, 30, 90, 180, 300) { + # Clear Maven's cached resolution failures first; see the first retry loop above. + Get-ChildItem -Path "$env:USERPROFILE\.m2\repository" -Recurse -File -Filter '*.lastUpdated' -ErrorAction SilentlyContinue | + Remove-Item -Force -ErrorAction SilentlyContinue if ($delay -gt 0) { Write-Host "mvn failed; retrying in $delay s..." Start-Sleep -Seconds $delay @@ -489,9 +512,12 @@ jobs: if ($LASTEXITCODE -eq 0) { $ok = $true; break } } if (-not $ok) { throw "mvn clean package failed after all retries" } - $resolutionFailure = 'status: (403|429|50[0-9])|Could not transfer artifact|Unresolveable build extension|Non-resolvable import POM|Could not resolve dependencies' + $resolutionFailure = 'status: (403|429|50[0-9])|Could not transfer artifact|Unresolveable build extension|Non-resolvable import POM|Could not resolve dependencies|UnknownHostException|Could not resolve host|Temporary failure in name resolution' $ok = $false foreach ($delay in 0, 30, 90, 180, 300) { + # Clear Maven's cached resolution failures first; see the first retry loop above. + Get-ChildItem -Path "$env:USERPROFILE\.m2\repository" -Recurse -File -Filter '*.lastUpdated' -ErrorAction SilentlyContinue | + Remove-Item -Force -ErrorAction SilentlyContinue if ($delay -gt 0) { Write-Host "mvn failed on a dependency-resolution error; retrying in $delay s..." Start-Sleep -Seconds $delay @@ -649,6 +675,9 @@ jobs: # blocking gate green. $ok = $false foreach ($delay in 0, 30, 90, 180, 300) { + # Clear Maven's cached resolution failures first; see the first retry loop above. + Get-ChildItem -Path "$env:USERPROFILE\.m2\repository" -Recurse -File -Filter '*.lastUpdated' -ErrorAction SilentlyContinue | + Remove-Item -Force -ErrorAction SilentlyContinue if ($delay -gt 0) { Write-Host "mvn failed; retrying in $delay s..." Start-Sleep -Seconds $delay @@ -657,9 +686,12 @@ jobs: if ($LASTEXITCODE -eq 0) { $ok = $true; break } } if (-not $ok) { throw "mvn clean package failed after all retries" } - $resolutionFailure = 'status: (403|429|50[0-9])|Could not transfer artifact|Unresolveable build extension|Non-resolvable import POM|Could not resolve dependencies' + $resolutionFailure = 'status: (403|429|50[0-9])|Could not transfer artifact|Unresolveable build extension|Non-resolvable import POM|Could not resolve dependencies|UnknownHostException|Could not resolve host|Temporary failure in name resolution' $ok = $false foreach ($delay in 0, 30, 90, 180, 300) { + # Clear Maven's cached resolution failures first; see the first retry loop above. + Get-ChildItem -Path "$env:USERPROFILE\.m2\repository" -Recurse -File -Filter '*.lastUpdated' -ErrorAction SilentlyContinue | + Remove-Item -Force -ErrorAction SilentlyContinue if ($delay -gt 0) { Write-Host "mvn failed on a dependency-resolution error; retrying in $delay s..." Start-Sleep -Seconds $delay diff --git a/.github/workflows/windows-cross-compile.yml b/.github/workflows/windows-cross-compile.yml index b2d5e05a41e..0e79d54ee9c 100644 --- a/.github/workflows/windows-cross-compile.yml +++ b/.github/workflows/windows-cross-compile.yml @@ -121,7 +121,7 @@ jobs: # retries reuse the previous attempt's target directories, so a partial output # can decide the result. Matching the output keeps a real failure terminal on # its first occurrence, and the retry starts from clean. - resolution='status: (403|429|50[0-9])|Could not transfer artifact|Failed to read artifact descriptor|Unresolveable build extension|Non-resolvable import POM|Could not resolve dependencies' + resolution='status: (403|429|50[0-9])|Could not transfer artifact|Failed to read artifact descriptor|Unresolveable build extension|Non-resolvable import POM|Could not resolve dependencies|UnknownHostException|Could not resolve host|Temporary failure in name resolution' goal=install for delay in 30 120 300 0; do # PIPESTATUS, not the pipeline's status: tee succeeds even when mvn does not, diff --git a/CodenameOne/src/com/codename1/annotations/OpenTelemetry.java b/CodenameOne/src/com/codename1/annotations/OpenTelemetry.java new file mode 100644 index 00000000000..65c81bdea05 --- /dev/null +++ b/CodenameOne/src/com/codename1/annotations/OpenTelemetry.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Traces the app with OpenTelemetry: every network request becomes a span, and +/// carries the W3C trace context to the server it reaches, so a tap in the app and +/// the backend work it caused are one trace. +/// +/// Put it on the main class. The build generates a bootstrap that installs +/// `com.codename1.telemetry.Telemetry` before the app starts, and nothing else in +/// the app changes. +/// +/// ```java +/// // Through the app's own Codename One backend (cn1.otel.relay=true there), +/// // which holds the collector's credentials: +/// @OpenTelemetry(relay = "https://api.example.com", serviceName = "shop-app") +/// public class ShopApp extends Lifecycle { ... } +/// +/// // Or straight to a collector. The header ships inside the app, so use a token +/// // that can write traces and nothing else: +/// @OpenTelemetry(endpoint = "https://collector.example.com:4318", +/// headers = "Authorization: Api-Token dt0c01.ingest-only") +/// public class ShopApp extends Lifecycle { ... } +/// ``` +/// +/// Exactly one of `relay` and `endpoint` is set. Without the annotation the +/// telemetry classes are never referenced, so the app does not carry them. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.TYPE) +public @interface OpenTelemetry { + /// The base URL of a Codename One backend that relays spans to the collector. + String relay() default ""; + + /// The base URL of an OTLP/HTTP collector to export to directly; + /// `/v1/traces` is appended. + String endpoint() default ""; + + /// The `service.name` the app reports as. Defaults to the app's name. + String serviceName() default ""; + + /// Headers for a direct export, each `"Name: value"`. + String[] headers() default {}; + + /// The token a relay configured with `cn1.otel.relay.token` expects. + String relayToken() default ""; + + /// The share of new traces recorded, from 0 to 1. + double sampleRatio() default 1.0; + + /// Whether a direct export is binary protobuf (the default) or JSON. + boolean protobuf() default true; + + /// Further hosts to send the trace context to. See + /// `TelemetryConfig.propagateTo`: on the web only the relay's host and these + /// receive it, because the header needs CORS permission. + String[] propagateTo() default {}; + + /// Traces only while the user has granted analytics consent through + /// `com.codename1.analytics.Analytics`. See + /// `TelemetryConfig.requireAnalyticsConsent`. + boolean requireAnalyticsConsent() default false; +} diff --git a/CodenameOne/src/com/codename1/io/ConnectionRequest.java b/CodenameOne/src/com/codename1/io/ConnectionRequest.java index 8c2d780405f..88296c90945 100644 --- a/CodenameOne/src/com/codename1/io/ConnectionRequest.java +++ b/CodenameOne/src/com/codename1/io/ConnectionRequest.java @@ -214,6 +214,55 @@ public class ConnectionRequest implements IOProgressListener { private boolean contentTypeSetExplicitly; private Object _connection; + /// What the [NetworkTracer] returned when this request was queued: the context + /// its spans are children of. Kept across retries, which are the same request. + /// + /// The tracer fields are per REQUEST OBJECT, like every other field of one + /// execution here -- the URL, the response code, the streams, the guard's + /// capture. The same instance queued to run twice at once (duplicates are + /// allowed by default) already has its two runs overwrite each other's response + /// state; tracing is no more per-execution than the request it observes, and + /// is not the place to make it so. Queue separate instances to run in parallel. + Object tracerParent; + + /// The tracer that produced [#tracerParent]. The context is that tracer's own + /// state, so it is handed only to that tracer: one installed between queuing and + /// running gets no parent rather than an object it cannot interpret -- or, for a + /// telemetry reinstall, a trace id belonging to the previous installation. + NetworkTracer tracerParentOwner; + + /// The attempt in flight, as the tracer's own state; null when none is being + /// traced. Set on the network thread and cleared there when the attempt ends. + Object tracerAttempt; + + /// The tracer that started [#tracerAttempt], which is the one that ends it. The + /// slot can be replaced or emptied while the attempt is in flight, and handing + /// one tracer's state to another -- or to none -- would leak it and lose the span. + NetworkTracer tracerOwner; + + /// Whether this attempt received a status line. Set the moment the status is + /// read, because a followed redirect and a 304 revalidation both return before + /// the guard's capture runs, and reporting them as "no response" hid the very + /// 3xx that explains the attempt. + boolean tracerResponded; + + /// The network thread running [#tracerAttempt]; only it may end the attempt. + Thread tracerThread; + + /// The last attempt that ended, and its tracer: the parent a retry of a request + /// queued with no context continues from, so the attempts share one trace. + Object tracerLastAttempt; + NetworkTracer tracerLastOwner; + + /// Whether [#tracerParent] is such an earlier attempt rather than the context + /// the request was queued under; a later retry then moves it to the newest one. + boolean tracerParentChained; + + /// A generation, advanced by every accepted enqueue -- a retry, a redirect or a + /// fresh reuse. An attempt that ends with it unchanged was the request's last, + /// and a cleanup queued for one generation never touches the next. + int tracerRequeues; + /// Default constructor public ConnectionRequest() { if (NetworkManager.getInstance().isAPSupported()) { @@ -753,6 +802,63 @@ && equalsIgnoreAsciiCase(existing, key) } } + /// Adds a header only when the request does not already carry one of that name, in any + /// spelling -- the counterpart of [#removeRequestHeaderIfUnchanged(String, String)] for + /// a layer that decorates a request it does not own. A tracer adding `traceparent` is + /// the case it exists for: when the app set its own, that is a deliberate choice of + /// which trace the request belongs to, and replacing it would move the request into + /// another one. + /// + /// #### Parameters + /// + /// - `key`: the header key, matched without regard to case as HTTP requires + /// + /// - `value`: the header value + /// + /// #### Returns + /// + /// true when the header was added, false when one was already there + public boolean addRequestHeaderIfAbsent(String key, String value) { + if (key == null || value == null || getRequestHeader(key) != null) { + return false; + } + addRequestHeader(key, value); + return true; + } + + /// The value of a header added to this request, matched without regard to case as + /// HTTP requires, or null when there is none. `Content-Type` answers only when it was + /// set explicitly: [#addRequestHeader(String, String)] routes that one to a dedicated + /// field, and the default it otherwise carries is not something anyone added. + /// + /// #### Parameters + /// + /// - `key`: the header name + /// + /// #### Returns + /// + /// the value, or null + public String getRequestHeader(String key) { + if (key == null) { + return null; + } + if ("content-type".equalsIgnoreCase(key)) { + return contentTypeSetExplicitly ? contentType : null; + } + if (userHeaders != null) { + Enumeration keys = userHeaders.keys(); + while (keys.hasMoreElements()) { + String existing = (String) keys.nextElement(); + if (existing != null && existing.length() == key.length() + && equalsIgnoreAsciiCase(existing, key)) { + Object value = userHeaders.get(existing); + return value == null ? null : value.toString(); + } + } + } + return null; + } + /// ASCII-only case-insensitive comparison, so the result never depends on the device locale -- /// under the Turkish locale an uppercase `I` does not fold to `i`. private static boolean equalsIgnoreAsciiCase(String a, String b) { @@ -784,11 +890,32 @@ void addRequestHeaderDontRepleace(String key, String value) { if (userHeaders == null) { userHeaders = new Hashtable(); } + // In any spelling, as HTTP matches names. An exact-key check let a default + // "traceparent" go out beside the request's own "Traceparent" -- two trace + // contexts on one request, which a server may resolve either way. Content-Type + // keeps the exact check: it lives in its own field, not in userHeaders, and + // getRequestHeader answers for it only when it was set explicitly. + if (key == null || (!"content-type".equalsIgnoreCase(key) && getRequestHeader(key) != null)) { + return; + } if (!userHeaders.containsKey(key)) { userHeaders.put(key, value); } } + /// Whether the headers added with [NetworkManager#addDefaultHeader(String, String)] + /// are sent with this request. They are meant for the app's own services -- an + /// `Authorization` for its backend is the usual one -- so a request that goes + /// somewhere else, a third-party collector for instance, overrides this to + /// keep them from being disclosed there. + /// + /// #### Returns + /// + /// true, the default: every request carries the default headers + protected boolean shouldApplyDefaultHeaders() { + return true; + } + void prepare() { complete = false; timeSinceLastUpdate = System.currentTimeMillis(); @@ -1143,6 +1270,25 @@ boolean performOperationComplete() throws IOException { // blocking token fetch would stall every other request. requestGuard.beforeRequest(this); } + tracerResponded = false; + NetworkTracer tracer = NetworkManager.getNetworkTracer(); + if (tracer != null) { + // After the guard, so the attempt the tracer times is the one that is + // really made, and in the same place for the same reason: before + // initConnection() writes the headers, and outside it so a subclass + // that overrides it cannot drop the trace context. NetworkManager's + // default headers are already on the request by now -- NetworkThread + // copies them before it calls runCurrentRequest -- so a traceparent the + // app supplies as a default is seen here and kept, not replaced. + try { + tracerAttempt = tracer.beforeRequest(this, + tracer == tracerParentOwner ? tracerParent : null); //NOPMD CompareObjectsWithEquals + tracerOwner = tracerAttempt == null ? null : tracer; + tracerThread = Thread.currentThread(); + } catch (Throwable t) { + Log.e(t); + } + } CodenameOneImplementation impl = Util.getImplementation(); Object connection = null; @@ -1271,6 +1417,7 @@ boolean performOperationComplete() throws IOException { } timeSinceLastUpdate = System.currentTimeMillis(); responseCode = impl.getResponseCode(connection); + tracerResponded = true; if (isCookiesEnabled()) { String[] cookies = impl.getHeaderFields("Set-Cookie", connection); diff --git a/CodenameOne/src/com/codename1/io/NetworkManager.java b/CodenameOne/src/com/codename1/io/NetworkManager.java index 8ebcf281d6c..18c22d5d8ae 100644 --- a/CodenameOne/src/com/codename1/io/NetworkManager.java +++ b/CodenameOne/src/com/codename1/io/NetworkManager.java @@ -239,6 +239,23 @@ public static synchronized NetworkGuard getNetworkGuard() { return networkGuard; } + /// Read through [#getNetworkTracer()], for the same publication reason as the guard. + private static NetworkTracer networkTracer; + + /// Installs the app-wide [NetworkTracer], replacing any earlier one; null removes it. + /// + /// Unlike the guard this slot does not seal: a tracer only observes, so replacing + /// one cannot weaken anything, and telemetry that is switched off at run time has + /// to be able to take itself out. + public static synchronized void setNetworkTracer(NetworkTracer tracer) { + networkTracer = tracer; + } + + /// The installed tracer, or null. + public static synchronized NetworkTracer getNetworkTracer() { + return networkTracer; + } + /// Test hook: drops the installed guard and unseals the slot. static void resetNetworkGuardForTesting() { synchronized (NetworkManager.class) { @@ -628,12 +645,127 @@ void kill9(final ConnectionRequest request) { } } + /// Ends the tracer attempt in flight on `req`, if any, with the tracer that + /// started it. Cleared first, so an attempt is ended exactly once however the + /// tracer behaves. + /// + /// Only the thread that started the attempt ends it. Once a retry has + /// re-queued the request, another worker may already have begun the NEXT + /// attempt by the time this one reaches its finally, and that attempt is + /// not this thread's to end. + static void endTracerAttempt(ConnectionRequest req, Throwable failure) { + Object attempt = req.tracerAttempt; + NetworkTracer owner = req.tracerOwner; + if (attempt == null || owner == null + || req.tracerThread != Thread.currentThread()) { //NOPMD CompareObjectsWithEquals + return; + } + req.tracerAttempt = null; + req.tracerOwner = null; + req.tracerThread = null; + // Kept for a retry of a request queued with no parent: see addToQueue. + req.tracerLastAttempt = attempt; + req.tracerLastOwner = owner; + try { + // Only a status THIS attempt received: a reused request still holds + // the last one's. + owner.afterRequest(req, attempt, + req.tracerResponded ? req.getResponseCode() : -1, failure); + } catch (Throwable t) { + // Observation must never change a request's outcome. + Log.e(t); + } + } + + /// Clears `req`'s tracer state on the EDT, after whatever this attempt already + /// queued there, unless the request was queued again in the meantime. + private static void scheduleTracerClear(final ConnectionRequest req, final int requeues) { + if (!Display.isInitialized()) { + clearTracerState(req); + return; + } + Display.getInstance().callSerially(new Runnable() { + @Override + public void run() { + if (req.tracerRequeues == requeues) { + clearTracerState(req); + } + } + }); + } + + /// Forgets every tracer object a finished request holds. + static void clearTracerState(ConnectionRequest req) { + req.tracerParent = null; + req.tracerParentOwner = null; + req.tracerParentChained = false; + req.tracerLastAttempt = null; + req.tracerLastOwner = null; + } + /// Adds the given network connection to the queue of execution /// /// #### Parameters /// /// - `request`: network request for execution void addToQueue(@Async.Schedule ConnectionRequest request, boolean retry) { + if (retry) { + // A redirect or retry re-queues THIS request object while its current + // attempt is still open on the worker that ran it. With more than one + // network thread another worker can pick it up before that worker + // reaches its finally and overwrite the attempt's state -- losing the + // span and leaving its traceparent on the request. So the attempt ends + // here, on the thread that ran it, before the request is visible to + // anyone else. + endTracerAttempt(request, null); + request.tracerRequeues++; + // A request queued with no context -- the usual case for a generated + // client used outside Telemetry.run -- would start a NEW trace on every + // attempt: a 302 and the 200 it led to, or a failure and the retry that + // succeeded, came out as unrelated traces with separate sampling + // decisions, and the logical request could not be followed. So the + // attempt that just ended becomes the next one's parent: one trace, + // one decision, each attempt still its own span. A request that WAS + // queued inside an action keeps that action as every attempt's parent. + // A parent some OTHER tracer captured is as good as none: the tracer + // that ran this attempt will not use it (it only takes its own), and + // keeping it blocked the chain, so every retry started a new root. + // The attempt to continue from is the last one that ENDED -- or, when a + // listener retries from the EDT before the network thread has finished + // the attempt it is reacting to, that attempt, still in flight. Waiting + // for "ended" alone lost the race on a fast EDT, and the retry started + // an unrelated trace. + Object previous = request.tracerAttempt != null + ? request.tracerAttempt : request.tracerLastAttempt; + NetworkTracer previousOwner = request.tracerAttempt != null + ? request.tracerOwner : request.tracerLastOwner; + if ((request.tracerParent == null || request.tracerParentChained + || request.tracerParentOwner != previousOwner) //NOPMD CompareObjectsWithEquals + && previous != null) { + request.tracerParent = previous; + request.tracerParentOwner = previousOwner; + request.tracerParentChained = true; + } + } + // Captured HERE, on the thread that asked for the request, so the span it + // becomes is a child of what the app was doing at the time. A retry keeps + // the context of the request it retries. Held in locals and stored only once + // the enqueue is accepted below: re-adding a request that is already pending + // is rejected as a duplicate, and storing first would re-parent the queued + // one under whatever the rejected call was doing. + NetworkTracer queuedBy = null; + Object queuedParent = null; + if (!retry) { + NetworkTracer tracer = getNetworkTracer(); + if (tracer != null) { + try { + queuedParent = tracer.requestQueued(request); + queuedBy = tracer; + } catch (Throwable t) { + Log.e(t); + } + } + } Util.getImplementation().addConnectionToQueue(request); if (!running) { start(); @@ -663,6 +795,17 @@ void addToQueue(@Async.Schedule ConnectionRequest request, boolean retry) { return; } } + request.tracerParent = queuedParent; + request.tracerParentOwner = queuedBy; + // A fresh enqueue is a new logical request, not a retry of the last. + // It advances the generation too: a cleanup the previous run queued + // on the EDT would otherwise still match, and clear the parent this + // enqueue just captured -- a listener can reuse a finished request + // with addToQueue before that cleanup runs. + request.tracerRequeues++; + request.tracerParentChained = false; + request.tracerLastAttempt = null; + request.tracerLastOwner = null; } else { i = ConnectionRequest.PRIORITY_HIGH; } @@ -1108,6 +1251,13 @@ private boolean runCurrentRequest(@Async.Execute ConnectionRequest req) { int frameRate = -1; boolean requestWasCompleted = true; + // What failed the attempt, for the tracer. Both catches below handle the + // failure and do not rethrow, so the finally is the one place that sees + // every ending. + Throwable failure = null; + // How many times the request had been re-queued when this attempt + // began; compared in the finally to learn whether it was the last. + int requeuesBefore = req.tracerRequeues; // Default this to true because if, for some reason an exception is thrown // before calling performOperationComplete(), then the request // won't be retried. @@ -1147,6 +1297,12 @@ private boolean runCurrentRequest(@Async.Execute ConnectionRequest req) { requestWasCompleted = req.performOperationComplete(); } catch (IOException e) { + failure = e; + // Ended HERE, with the failure, before any handler runs: a handler + // that retries re-queues the request, and ending the attempt at + // that point has no failure to report, so the span came out with + // neither a response nor an error. + endTracerAttempt(req, e); if (!req.isFailSilently()) { if (!handleException(req, e)) { req.handleIOException(e); @@ -1156,6 +1312,8 @@ private boolean runCurrentRequest(@Async.Execute ConnectionRequest req) { Log.e(e); } } catch (RuntimeException er) { + failure = er; + endTracerAttempt(req, er); if (!req.isFailSilently()) { if (!handleException(req, er)) { req.handleRuntimeException(er); @@ -1172,6 +1330,24 @@ private boolean runCurrentRequest(@Async.Execute ConnectionRequest req) { if (requestWasCompleted) { req.complete = true; } + endTracerAttempt(req, failure); + // Any tracer field, the owners included: a request queued outside an + // action holds only tracerParentOwner, and that alone pins the whole + // telemetry installation. + if (req.tracerRequeues == requeuesBefore + && (req.tracerParent != null || req.tracerLastAttempt != null + || req.tracerParentOwner != null || req.tracerLastOwner != null)) { + // Nothing queued this request again YET. Its tracer state has + // to go once it is done -- the parent and the last attempt are + // the tracer's own objects, a span and through it the whole + // installation, and a request an app keeps for reuse held them + // for as long as it lived. But not from here: an exception or + // response-code listener runs LATER, on the EDT, and may still + // call retry(), which needs the last attempt to continue its + // trace. So the clear is queued on the EDT behind those + // listener callbacks, and skipped if one of them retried. + scheduleTracerClear(req, req.tracerRequeues); + } NetworkGuard guard = getNetworkGuard(); if (guard != null && req.hasGuardResponse()) { try { @@ -1225,6 +1401,17 @@ public void run() { pending.removeElementAt(0); currentRequest.prepare(); if (currentRequest.isKilled()) { + // Killed while it waited: runCurrentRequest, whose + // finally forgets the tracer state addToQueue + // captured, never runs -- so forget it here, or a + // request the app keeps holds the parent span and + // through it the whole telemetry installation. And + // let go of the request itself: the worker would + // otherwise hold it as currentRequest until the + // next one arrives. + scheduleTracerClear(currentRequest, currentRequest.tracerRequeues); + currentRequest = null; + LOCK.notifyAll(); continue; } currentRequest.setId(nextConnectionId++); @@ -1232,7 +1419,7 @@ public void run() { nextConnectionId = 1; } } - if (userHeaders != null) { + if (userHeaders != null && currentRequest.shouldApplyDefaultHeaders()) { Enumeration e = userHeaders.keys(); while (e.hasMoreElements()) { String key = (String) e.nextElement(); diff --git a/CodenameOne/src/com/codename1/io/NetworkTracer.java b/CodenameOne/src/com/codename1/io/NetworkTracer.java new file mode 100644 index 00000000000..a6cb8072beb --- /dev/null +++ b/CodenameOne/src/com/codename1/io/NetworkTracer.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.io; + +/// Observes every [ConnectionRequest] the [NetworkManager] runs, so a tracer can time +/// each one and put trace context on it. +/// +/// Installed with [NetworkManager#setNetworkTracer(NetworkTracer)]. The one +/// implementation is `com.codename1.telemetry.Telemetry`'s, and it is reached only +/// from the bootstrap the build generates for a project that enables it -- so an app +/// that does not trace carries this interface and nothing else. +/// +/// This is a separate slot from [NetworkGuard] on purpose. The guard is a security +/// decision: it seals on first install and can veto a request. A tracer observes and +/// decorates, never refuses, and must not have to compete with the guard for the one +/// slot the guard deliberately makes unreplaceable. +/// +/// Every callback is guarded by the caller: an exception thrown from one is logged and +/// the request carries on untraced. +public interface NetworkTracer { + /// A request is being queued, on the thread that queued it -- the EDT, for most + /// requests. Whatever this returns is handed back to [#beforeRequest], which is + /// how a request records the span that was current when the app ASKED for it, + /// rather than whatever is current on the network thread later. + /// + /// #### Returns + /// + /// the parent context, or null + Object requestQueued(ConnectionRequest request); + + /// An attempt is about to connect, on the network thread, before the request's + /// headers are written. Headers added here with + /// [ConnectionRequest#addRequestHeader(String, String)] are sent. Called again + /// for every retry and redirect, each of which is its own attempt. + /// + /// #### Parameters + /// + /// - `request`: the request + /// + /// - `parent`: what [#requestQueued] returned for it -- or, for a retry or + /// redirect of a request queued with no parent, the attempt before it (what + /// this method returned then), so that the attempts share one trace + /// + /// #### Returns + /// + /// the attempt's state, handed to [#afterRequest], or null to not trace it + Object beforeRequest(ConnectionRequest request, Object parent); + + /// An attempt [#beforeRequest] started has ended, on the network thread. + /// + /// #### Parameters + /// + /// - `request`: the request + /// + /// - `attempt`: what [#beforeRequest] returned + /// + /// - `responseCode`: the HTTP status, or a value below 100 when none arrived + /// + /// - `error`: what failed the attempt, or null + void afterRequest(ConnectionRequest request, Object attempt, int responseCode, Throwable error); +} diff --git a/CodenameOne/src/com/codename1/telemetry/OtlpEncoding.java b/CodenameOne/src/com/codename1/telemetry/OtlpEncoding.java new file mode 100644 index 00000000000..28a18262e8e --- /dev/null +++ b/CodenameOne/src/com/codename1/telemetry/OtlpEncoding.java @@ -0,0 +1,394 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.telemetry; + +import com.codename1.util.StringUtil; + +import java.util.List; +import java.util.Map; + +/// OTLP's `ExportTraceServiceRequest`, written from spans in either of its two +/// encodings. +/// +/// Hand written because a protobuf runtime is far larger than the dozen fields a +/// trace export uses, and an app that turns telemetry on should not grow by a +/// library to do it. The field numbers are opentelemetry-proto's `trace.proto`, +/// `common.proto` and `resource.proto`; the tests decode both encodings with the +/// classes generated from those files. +final class OtlpEncoding { + static final String SCOPE_NAME = "com.codename1.telemetry"; + + private OtlpEncoding() { + } + + // ------------------------------------------------------------------ + // JSON + // ------------------------------------------------------------------ + + static byte[] json(Map resource, List spans) { + StringBuilder b = new StringBuilder(256 + spans.size() * 256); + b.append("{\"resourceSpans\":[{\"resource\":{\"attributes\":"); + jsonAttributes(b, resource); + b.append("},\"scopeSpans\":[{\"scope\":{\"name\":\"").append(SCOPE_NAME) + .append("\"},\"spans\":["); + boolean first = true; + for (TelemetrySpan span : spans) { + if (!first) { + b.append(','); + } + first = false; + jsonSpan(b, span); + } + b.append("]}]}]}"); + return utf8(b.toString()); + } + + private static void jsonSpan(StringBuilder b, TelemetrySpan span) { + b.append("{\"traceId\":\"").append(span.traceId) + .append("\",\"spanId\":\"").append(span.spanId).append('"'); + if (span.parentSpanId != null) { + b.append(",\"parentSpanId\":\"").append(span.parentSpanId).append('"'); + } + b.append(",\"flags\":").append(flags(span)); + b.append(",\"name\":"); + jsonString(b, span.name); + b.append(",\"kind\":").append(span.kind); + // Strings: proto3's JSON mapping writes 64-bit integers that way, since + // a nanosecond timestamp is past what a JavaScript number holds exactly. + b.append(",\"startTimeUnixNano\":\"").append(span.startEpochNanos).append('"'); + b.append(",\"endTimeUnixNano\":\"").append(span.endEpochNanos).append('"'); + b.append(",\"attributes\":"); + jsonAttributes(b, span.attributes); + if (span.droppedAttributes > 0) { + b.append(",\"droppedAttributesCount\":").append(span.droppedAttributes); + } + if (span.events != null && !span.events.isEmpty()) { + b.append(",\"events\":["); + boolean first = true; + for (Object[] event : span.events) { + if (!first) { + b.append(','); + } + first = false; + b.append("{\"timeUnixNano\":\"").append(event[0]).append("\",\"name\":"); + jsonString(b, String.valueOf(event[1])); + b.append(",\"attributes\":"); + jsonAttributes(b, asMap(event[2])); + b.append('}'); + } + b.append(']'); + } + if (span.droppedEvents > 0) { + b.append(",\"droppedEventsCount\":").append(span.droppedEvents); + } + if (span.statusCode != 0) { + b.append(",\"status\":{"); + if (span.statusMessage != null) { + b.append("\"message\":"); + jsonString(b, span.statusMessage); + b.append(','); + } + b.append("\"code\":").append(span.statusCode).append('}'); + } + b.append('}'); + } + + private static void jsonAttributes(StringBuilder b, Map attributes) { + b.append('['); + if (attributes != null) { + boolean first = true; + for (Map.Entry entry : attributes.entrySet()) { + if (!first) { + b.append(','); + } + first = false; + b.append("{\"key\":"); + jsonString(b, entry.getKey()); + b.append(",\"value\":{"); + Object v = entry.getValue(); + if (v instanceof Boolean) { + b.append("\"boolValue\":").append(v); + } else if (v instanceof Long || v instanceof Integer) { + b.append("\"intValue\":\"").append(v).append('"'); + } else if (v instanceof Double) { + b.append("\"doubleValue\":").append(v); + } else { + b.append("\"stringValue\":"); + jsonString(b, String.valueOf(v)); + } + b.append("}}"); + } + } + b.append(']'); + } + + private static void jsonString(StringBuilder b, String value) { + b.append('"'); + int n = value.length(); + for (int i = 0; i < n; i++) { + char c = value.charAt(i); + switch (c) { + case '"': + b.append("\\\""); + break; + case '\\': + b.append("\\\\"); + break; + case '\n': + b.append("\\n"); + break; + case '\r': + b.append("\\r"); + break; + case '\t': + b.append("\\t"); + break; + default: + if (c < 0x20) { + String hex = Integer.toHexString(c); + b.append("\\u"); + for (int pad = hex.length(); pad < 4; pad++) { + b.append('0'); + } + b.append(hex); + } else { + b.append(c); + } + break; + } + } + b.append('"'); + } + + // ------------------------------------------------------------------ + // Protobuf + // ------------------------------------------------------------------ + + static byte[] protobuf(Map resource, List spans) { + Sink scope = new Sink(); + Sink scopeName = new Sink(); + scopeName.string(1, SCOPE_NAME); + scope.message(1, scopeName); // ScopeSpans.scope + for (TelemetrySpan span : spans) { + scope.message(2, protoSpan(span)); // ScopeSpans.spans + } + Sink resourceMessage = new Sink(); + protoAttributes(resourceMessage, 1, resource); // Resource.attributes + Sink resourceSpans = new Sink(); + resourceSpans.message(1, resourceMessage); // ResourceSpans.resource + resourceSpans.message(2, scope); // ResourceSpans.scope_spans + Sink request = new Sink(); + request.message(1, resourceSpans); // ExportTraceServiceRequest.resource_spans + return request.toByteArray(); + } + + private static Sink protoSpan(TelemetrySpan span) { + Sink out = new Sink(); + out.bytes(1, hex(span.traceId)); + out.bytes(2, hex(span.spanId)); + if (span.parentSpanId != null) { + out.bytes(4, hex(span.parentSpanId)); + } + out.string(5, span.name); + out.varint(6, span.kind); + out.fixed64(7, span.startEpochNanos); + out.fixed64(8, span.endEpochNanos); + protoAttributes(out, 9, span.attributes); + if (span.droppedAttributes > 0) { + out.varint(10, span.droppedAttributes); + } + if (span.events != null) { + for (Object[] event : span.events) { + Sink e = new Sink(); + e.fixed64(1, ((Long) event[0]).longValue()); + e.string(2, String.valueOf(event[1])); + protoAttributes(e, 3, asMap(event[2])); + out.message(11, e); + } + } + if (span.droppedEvents > 0) { + out.varint(12, span.droppedEvents); + } + if (span.statusCode != 0) { + Sink status = new Sink(); + if (span.statusMessage != null) { + status.string(2, span.statusMessage); + } + status.varint(3, span.statusCode); + out.message(15, status); + } + out.fixed32(16, flags(span)); + return out; + } + + private static void protoAttributes(Sink out, int field, Map attributes) { + if (attributes == null) { + return; + } + for (Map.Entry entry : attributes.entrySet()) { + Sink value = new Sink(); + Object v = entry.getValue(); + if (v instanceof Boolean) { + value.varint(2, ((Boolean) v).booleanValue() ? 1 : 0); + } else if (v instanceof Long || v instanceof Integer) { + value.varint(3, ((Number) v).longValue()); + } else if (v instanceof Double) { + value.fixed64(4, Double.doubleToLongBits(((Double) v).doubleValue())); + } else { + value.string(1, String.valueOf(v)); + } + Sink kv = new Sink(); + kv.string(1, entry.getKey()); + kv.message(2, value); + out.message(field, kv); + } + } + + /// Trace flags in the low byte, then HAS_IS_REMOTE when there is a parent: + /// every parent the app has is local. A root has no parent context to + /// describe, so the bits stay clear ("unknown" in trace.proto) rather than + /// claiming a local parent it does not have. + private static int flags(TelemetrySpan span) { + int flags = span.sampled ? 1 : 0; + if (span.parentSpanId != null) { + flags |= 0x100; + } + return flags; + } + + private static Map asMap(Object value) { + if (value instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) value; + return map; + } + return null; + } + + private static byte[] hex(String text) { + byte[] out = new byte[text.length() / 2]; + for (int i = 0; i < out.length; i++) { + out[i] = (byte) ((digit(text.charAt(i * 2)) << 4) | digit(text.charAt(i * 2 + 1))); + } + return out; + } + + private static int digit(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + return c - 'A' + 10; + } + + /// UTF-8 through `String.getBytes`, which on ParparVM is the VM's native + /// vectorized encoder rather than a loop in Java. + static byte[] utf8(String value) { + return StringUtil.getBytes(value); + } + + /// A protobuf message being written. Nested messages are written into a sink + /// of their own first, because a length-delimited field needs its length + /// before its bytes. + static final class Sink { + private byte[] data = new byte[64]; + private int length; + + private void ensure(int extra) { + if (length + extra > data.length) { + int size = data.length * 2; + while (size < length + extra) { + size *= 2; + } + byte[] grown = new byte[size]; + System.arraycopy(data, 0, grown, 0, length); + data = grown; + } + } + + void tag(int field, int wireType) { + rawVarint(((long) field << 3) | wireType); + } + + void rawVarint(long value) { + long v = value; + ensure(10); + while ((v & ~0x7fL) != 0) { + data[length++] = (byte) ((v & 0x7f) | 0x80); + v >>>= 7; + } + data[length++] = (byte) v; + } + + void varint(int field, long value) { + tag(field, 0); + rawVarint(value); + } + + void fixed64(int field, long value) { + tag(field, 1); + ensure(8); + for (int i = 0; i < 8; i++) { + data[length++] = (byte) (value >> (8 * i)); + } + } + + void fixed32(int field, int value) { + tag(field, 5); + ensure(4); + for (int i = 0; i < 4; i++) { + data[length++] = (byte) (value >> (8 * i)); + } + } + + void bytes(int field, byte[] value) { + bytes(field, value, value.length); + } + + private void bytes(int field, byte[] value, int count) { + tag(field, 2); + rawVarint(count); + ensure(count); + System.arraycopy(value, 0, data, length, count); + length += count; + } + + void string(int field, String value) { + bytes(field, utf8(value)); + } + + /// A nested message, copied straight out of the child's buffer: its + /// length has to precede it, so it is written once the child is complete. + void message(int field, Sink child) { + bytes(field, child.data, child.length); + } + + byte[] toByteArray() { + byte[] out = new byte[length]; + System.arraycopy(data, 0, out, 0, length); + return out; + } + } +} diff --git a/CodenameOne/src/com/codename1/telemetry/Telemetry.java b/CodenameOne/src/com/codename1/telemetry/Telemetry.java new file mode 100644 index 00000000000..90f4b560b54 --- /dev/null +++ b/CodenameOne/src/com/codename1/telemetry/Telemetry.java @@ -0,0 +1,913 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.telemetry; + +import com.codename1.analytics.Analytics; +import com.codename1.analytics.AnalyticsConsent; +import com.codename1.analytics.ConsentMode; +import com.codename1.io.ConnectionRequest; +import com.codename1.io.Data; +import com.codename1.io.Log; +import com.codename1.io.NetworkManager; +import com.codename1.io.NetworkTracer; +import com.codename1.security.Hash; +import com.codename1.security.SecureRandom; +import com.codename1.ui.CN; +import com.codename1.ui.Display; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Timer; + +/// OpenTelemetry tracing for the app: every network request is a span, and carries +/// W3C trace context to the server it reaches, so the app's request and the +/// backend's handling of it are one trace. +/// +/// Usually installed by the build rather than by hand -- put +/// `@OpenTelemetry` on the main class and the generated bootstrap calls +/// [#install(TelemetryConfig)] before the app starts: +/// +/// ```java +/// @OpenTelemetry(relay = "https://api.example.com", serviceName = "shop-app") +/// public class ShopApp extends Lifecycle { ... } +/// ``` +/// +/// Every [ConnectionRequest] is covered, which is every REST, gRPC-Web and GraphQL +/// client the build generates. To group the requests a user action causes, time +/// the action: +/// +/// ```java +/// Telemetry.run("checkout", () -> cart.submit()); +/// ``` +/// +/// Spans are batched and exported with OTLP/HTTP -- through the app's own backend +/// by default, which keeps the collector's credentials out of the app (see +/// [TelemetryConfig]). +public final class Telemetry { + private static final String TRACEPARENT = "traceparent"; + private static final String TRACESTATE = "tracestate"; + /// How many export batches may wait in the network queue at once. + static final int MAX_PENDING_EXPORTS = 2; + /// The installation. A plain field, deliberately, like the tracer and guard + /// slots in NetworkManager it fills: install and uninstall are lifecycle + /// calls -- the generated bootstrap's, before Display.init, or the app's, on + /// the EDT -- and Codename One core does not synchronize framework state + /// (PMD's AvoidUsingVolatile gate says the same). A network thread or a task + /// started after the install sees it through Thread.start's happens-before + /// edge. Reinstalling telemetry while background tasks are mid-flight is not + /// a supported pattern, and locking every span start to serve it would tax + /// every request of every app that never does it. + private static State state; + /// The span [#run(String, Runnable)] has made current, per thread. Mostly the + /// EDT's, since that is where a user action runs and where requests are queued, + /// but `run` is public and a background task may time its own work: one static + /// slot let two overlapping tasks adopt and restore each other's spans. + private static final ThreadLocal CURRENT = new ThreadLocal(); + + private Telemetry() { + } + + /// Installs telemetry, replacing any earlier installation. + /// + /// Safe to call before `Display.init`, which is where the generated bootstrap + /// calls it: nothing that needs the platform -- not even the secure random + /// source the ids come from -- is touched until the first span. + /// + /// #### Parameters + /// + /// - `config`: where and how to export; a configuration with no endpoint + /// installs nothing + public static void install(TelemetryConfig config) { + uninstall(); + if (config == null || config.exportUrl() == null) { + Log.p("Telemetry: no endpoint configured, so no spans are recorded"); + return; + } + State installed = new State(config); + state = installed; + NetworkManager.setNetworkTracer(installed); + } + + /// Stops recording, exports what is buffered, and removes the network hook. + public static void uninstall() { + State old = state; + state = null; + CURRENT.remove(); + if (old != null) { + // Only if the slot still holds OURS: an app may have replaced the + // generated tracer with its own, and uninstalling telemetry must not + // silently switch that one off. + if (NetworkManager.getNetworkTracer() == old) { //NOPMD CompareObjectsWithEquals -- identity: is the slot still THIS installation + NetworkManager.setNetworkTracer(null); + } + old.stop(); + } + } + + /// Whether telemetry is installed. + public static boolean isInstalled() { + return state != null; + } + + /// A new span, a child of the current one. It is not made current; the caller + /// must end it. Never null -- with telemetry off it records nothing. + /// + /// #### Parameters + /// + /// - `name`: what the span times + public static TelemetrySpan startSpan(String name) { + State s = state; + TelemetrySpan span = s == null || !s.permitted() ? null + : s.start(name, TelemetrySpan.KIND_INTERNAL, CURRENT.get()); + return span != null ? span : new TelemetrySpan(null, name, TelemetrySpan.KIND_INTERNAL, + "00000000000000000000000000000000", "0000000000000000", null, false); + } + + /// Runs `work` inside a new span, which is current while it runs: requests + /// queued inside it become its children. A RuntimeException is recorded on + /// the span and rethrown. + /// + /// #### Parameters + /// + /// - `name`: what the span times + /// + /// - `work`: the work + public static void run(String name, Runnable work) { + TelemetrySpan span = startSpan(name); + TelemetrySpan previous = CURRENT.get(); + CURRENT.set(span); + try { + work.run(); + } catch (RuntimeException err) { + span.recordException(err); + throw err; + } finally { + CURRENT.set(previous); + span.end(); + } + } + + /// The span [#run(String, Runnable)] made current on this thread, or null. + public static TelemetrySpan getCurrentSpan() { + return CURRENT.get(); + } + + /// Exports what is buffered now, rather than at the next interval. Call it + /// when the app is paused: a span still in memory when the process is + /// reclaimed is lost. + public static void flush() { + State s = state; + if (s != null) { + s.flush(); + } + } + + /// Everything one installation owns. It is the [NetworkTracer] too, so the + /// network thread reaches the same configuration the app installed. + /// Not final so a test can stand in for the network queue: whether exports are + /// backed up is a property of the shared NetworkManager, which a test cannot hold + /// still reliably. + static class State implements NetworkTracer { + private final TelemetryConfig config; + private final String exportUrl; + /// The relay's ORIGIN -- scheme, host and effective port -- which is what + /// the browser keys CORS on. The host alone let a request to another port + /// or scheme on the same host through, and a traceparent can turn a request + /// the browser would have sent as-is into a preflight that fails. + private final String backendOrigin; + /// Touched on the EDT only: spans that end elsewhere are marshalled there. + final List buffer = new ArrayList(); + private Timer timer; + private Map resource; + private boolean stopped; + /// Set once, the first time the platform refuses secure random bytes. + private boolean idsUnavailable; + + State(TelemetryConfig config) { + // A snapshot: see TelemetryConfig.copy. + this.config = config.copy(); + this.exportUrl = config.exportUrl(); + this.backendOrigin = config.mode == TelemetryConfig.Mode.RELAY ? origin(exportUrl) : null; + } + + /// A new span, or null when ids cannot be made on this platform. + TelemetrySpan start(String name, int kind, TelemetrySpan parent) { + String traceId; + String parentId = null; + boolean sampled; + byte[] spanBytes = random(8); + if (spanBytes == null) { + return null; + } + // Only a parent THIS installation recorded. A thread still inside + // run() across an uninstall and reinstall holds the old one's span, and + // inheriting it would file the new installation's spans under the old + // trace, with the old sampling decision, possibly at another collector. + if (parent != null && parent.isOwnedBy(this) && parent.traceId.length() == 32 + && !isZero(parent.spanId)) { + traceId = parent.traceId; + parentId = parent.spanId; + // Follow the parent's decision, so a trace is whole or absent. + sampled = parent.sampled; + } else { + byte[] traceBytes = random(16); + if (traceBytes == null) { + return null; + } + traceId = Hash.toHex(traceBytes); + sampled = sample(traceBytes); + } + return new TelemetrySpan(this, name, kind, traceId, Hash.toHex(spanBytes), parentId, + sampled, parentId == null ? null : parent); + } + + /// Secure random bytes, or null when the platform has none. Asked here, + /// at the first span, rather than at install: install runs before + /// Display.init, where no platform exists yet to answer, and treating that + /// as "no random source" switched telemetry off on every device. + private byte[] random(int length) { + if (idsUnavailable) { + return null; + } + if (!Display.isInitialized()) { + // Too early to ask, not a platform without a source: before + // Display.init there is no implementation behind SecureRandom. Not + // cached, or a span a program started during start-up switched + // telemetry off for good once the display did exist. + return null; + } + try { + return SecureRandom.bytes(length); + } catch (RuntimeException err) { + idsUnavailable = true; + Log.p("Telemetry: this platform has no secure random source, so it cannot make " + + "trace ids; no spans are recorded"); + return null; + } + } + + /// Whether tracing may run now: always, unless the configuration asked for + /// analytics consent, in which case the recorded choice -- or, before the + /// user has made one, the consent mode -- decides. Read each time, so a + /// choice the user changes takes effect on the next request. + boolean permitted() { + if (!config.requireAnalyticsConsent) { + return true; + } + if (!Display.isInitialized()) { + // No storage yet, so the saved choice cannot be read -- and reading + // it anyway made Analytics record "loaded, nothing saved" for the + // rest of the run, ignoring a persisted grant, or in opt-out mode a + // persisted denial. Nothing is traced until it can be asked. + return false; + } + AnalyticsConsent consent = Analytics.getConsent(); + if (consent != null) { + return consent.isAnalytics(); + } + return Analytics.getConsentMode() == ConsentMode.OPT_OUT; + } + + /// The ratio decision, from the id's low bytes as the other OpenTelemetry + /// SDKs take it, so a backend sampling at the same ratio agrees without + /// being told. + private boolean sample(byte[] traceId) { + if (config.sampleRatio >= 1) { + return true; + } + if (config.sampleRatio <= 0) { + return false; + } + long low = 0; + for (int i = 8; i < 16; i++) { + low = (low << 8) | (traceId[i] & 0xff); + } + long bound = (long) (config.sampleRatio * (double) Long.MAX_VALUE); + return (low & Long.MAX_VALUE) < bound; + } + + /// From any thread, as a span ends. The buffer is the EDT's, so the span + /// is handed over there rather than guarded where it is. + void ended(final TelemetrySpan span) { + if (stopped) { + return; + } + if (!Display.isInitialized()) { + // Nowhere to deliver it yet; a span before the app exists is + // start-up noise, not what anyone is tracing. + return; + } + if (CN.isEdt()) { + record(span); + return; + } + CN.callSerially(new Runnable() { + @Override + public void run() { + recordHandedOff(span); + } + }); + } + + /// A span that ended on another thread and reaches the EDT only now. It + /// passed the stopped check where it ended, so it finished while this + /// installation was running; if an uninstall or a reinstall ran on the EDT + /// in between, it is exported on its own rather than discarded -- which + /// lost the request spans that were completing just as telemetry was + /// reconfigured. + void recordHandedOff(TelemetrySpan span) { + if (!stopped) { + record(span); + return; + } + if (!permitted()) { + return; + } + buffer.add(span); + if (buffer.size() > maxBuffered()) { + buffer.remove(0); + } + // ONE final flush for the burst, not one per span: every other late + // handoff already queued on the EDT runs before this, so they share a + // single export. A flush per span put an uncapped export -- and its + // encoded body -- in the network queue for each of them. + if (!lateFlushScheduled) { + lateFlushScheduled = true; + CN.callSerially(new Runnable() { + @Override + public void run() { + lateFlushScheduled = false; + flush(true); + } + }); + } + } + + /// Whether a final flush for late handoffs is already queued on the EDT. + private boolean lateFlushScheduled; + + void record(TelemetrySpan span) { + if (stopped || !permitted()) { + return; + } + buffer.add(span); + if (buffer.size() > maxBuffered()) { + // BOUNDED. While exports cannot keep up -- the collector is slow, or + // the app's own traffic keeps outranking them -- the oldest spans + // go, rather than the app's memory. + buffer.remove(0); + } + if (timer == null) { + // Started on the first span, on the EDT, because CN.setInterval + // needs the display and install() may run before there is one. + timer = CN.setInterval(config.flushIntervalMillis, new Runnable() { + @Override + public void run() { + flush(); + } + }); + } + if (buffer.size() >= config.batchSize) { + flush(); + } + } + + void flush() { + flush(false); + } + + /// `last` is the installation's final flush, from [#stop()]: it sends the + /// buffer even when exports are already waiting. Otherwise an uninstall or a + /// reinstall behind a full export queue left the buffer behind for good -- + /// the state stops, its timer is cancelled, and nothing would ever flush it + /// again. One extra batch, and the buffer it comes from is bounded. + void flush(final boolean last) { + if (!CN.isEdt()) { + if (Display.isInitialized()) { + CN.callSerially(new Runnable() { + @Override + public void run() { + flush(last); + } + }); + } + return; + } + if (buffer.isEmpty()) { + return; + } + if (!permitted()) { + // Consent was withdrawn after these were recorded: what the user + // refused is not sent, whenever it was collected. + buffer.clear(); + return; + } + if (!last && pendingExports() >= MAX_PENDING_EXPORTS) { + // Exports are already waiting in the network queue. Queuing another + // batch would hold one more byte array per flush for as long as the + // app's requests outrank them, with no limit; the spans stay in the + // bounded buffer and go out once the queue drains. + return; + } + List batch = new ArrayList(buffer); + buffer.clear(); + boolean json = config.mode == TelemetryConfig.Mode.RELAY || !config.protobuf; + byte[] body = json ? OtlpEncoding.json(resource(), batch) + : OtlpEncoding.protobuf(resource(), batch); + ExportRequest request = new ExportRequest(this, body); + request.setUrl(exportUrl); + request.setPost(true); + request.setHttpMethod("POST"); + if (config.mode == TelemetryConfig.Mode.DIRECT) { + for (String[] header : config.headers) { + request.addRequestHeader(header[0], header[1]); + } + } else if (config.relayToken != null && config.relayToken.length() > 0) { + request.addRequestHeader("X-CN1-Telemetry-Token", config.relayToken); + } + // AFTER the configured headers, so nothing among them can relabel the + // body: the config refuses a Content-Type, and this holds regardless. + request.setContentType(json ? "application/json" : "application/x-protobuf"); + // A failed export is dropped, never retried into the queue: telemetry + // must not compete with the app's own requests for the network. + request.setFailSilently(true); + request.setReadResponseForErrors(false); + // Never followed. A redirect re-queues this same request with its + // headers intact, so a collector that redirected elsewhere -- another + // origin included -- would be handed the Authorization or API-key + // header meant for it, and 301/302/303 turn the POST into a bodiless + // GET anyway. An export that is redirected fails, and is dropped like + // any other failed export; point the endpoint at the real collector. + request.setFollowRedirects(false); + // SHORT, and behind the app's own requests. By default a request may + // take five minutes and the manager has one network thread, so a + // collector that accepts the connection and never answers would hold + // every request the app makes behind an export nobody is waiting for. + request.setTimeout(10000); + request.setReadTimeout(10000); + request.setPriority(ConnectionRequest.PRIORITY_LOW); + NetworkManager.getInstance().addToQueue(request); + } + + /// Telemetry exports still waiting to be sent. Read from the queue itself + /// rather than counted, because a fail-silent export that fails reports + /// nothing back to count with. Any installation's exports count: after a + /// reinstall the previous one's are still competing for the same network. + int pendingExports() { + int count = 0; + java.util.Enumeration queue = NetworkManager.getInstance().enumurateQueue(); + while (queue.hasMoreElements()) { + if (queue.nextElement() instanceof ExportRequest) { + count++; + } + } + return count; + } + + private int maxBuffered() { + return Math.max(config.batchSize * 4, 128); + } + + void stop() { + flush(true); + stopped = true; + if (timer != null) { + timer.cancel(); + timer = null; + } + } + + /// The app, as the collector should see it. Built at the first export, + /// when the display that knows these things exists. + private Map resource() { + if (resource == null) { + Map out = new LinkedHashMap(); + String appName = Display.getInstance().getProperty("AppName", null); + // Trimmed, as the annotation's is: " " is no name, and exported + // as one the app had no usable service identity. + String configured = config.serviceName == null ? "" : config.serviceName.trim(); + String service = configured.length() > 0 ? configured : appName; + out.put("service.name", service == null || service.length() == 0 + ? "unknown_service" : service); + String version = Display.getInstance().getProperty("AppVersion", null); + if (version != null && version.length() > 0) { + out.put("service.version", version); + } + String platform = Display.getInstance().getPlatformName(); + if (platform != null) { + out.put("os.name", platform); + } + String osVersion = Display.getInstance().getProperty("OSVer", null); + if (osVersion != null && osVersion.length() > 0) { + out.put("os.version", osVersion); + } + out.put("telemetry.sdk.name", "codenameone"); + out.put("telemetry.sdk.language", "java"); + resource = out; + } + return resource; + } + + // -------------------------------------------------------------- + // NetworkTracer + // -------------------------------------------------------------- + + @Override + public Object requestQueued(ConnectionRequest request) { + return CURRENT.get(); + } + + @Override + public Object beforeRequest(ConnectionRequest request, Object parent) { + if (stopped || request instanceof ExportRequest || !permitted()) { + // The export's own request is not traced: each would be a span, + // and exporting that one another. + return null; + } + if (request.getRequestHeader(TRACEPARENT) != null + || request.getRequestHeader(TRACESTATE) != null) { + // The app chose which trace this request belongs to. The service it + // reaches joins THAT trace, so a span recorded here in another one + // would describe the same request twice, in two traces that never + // meet. The app's own instrumentation owns this request. A + // tracestate alone counts too: it is part of the app's context, + // and a traceparent of ours beside it paired the app's vendor state + // with an unrelated trace id. + return null; + } + String url = request.getUrl(); + String method = request.getHttpMethod(); + if (method == null || method.length() == 0) { + method = request.isPost() ? "POST" : "GET"; + } + TelemetrySpan span = start(method, TelemetrySpan.KIND_CLIENT, + parent instanceof TelemetrySpan ? (TelemetrySpan) parent : null); + if (span == null) { + return null; + } + String host = host(url); + if (span.isRecording()) { + span.setAttribute("http.request.method", method); + if (url != null) { + span.setAttribute("url.full", redact(url)); + } + if (host != null) { + span.setAttribute("server.address", host); + } + } + if (shouldPropagate(url, host)) { + // The app's own traceparent was ruled out above. Ours is removed + // again when the attempt ends (afterRequest), so a retry or a + // redirect starts clean -- the request object is reused, and a + // header left from an allowed host would otherwise follow a redirect + // to one that is not. + request.addRequestHeaderIfAbsent(TRACEPARENT, span.getTraceparent()); + } + return span; + } + + @Override + public void afterRequest(ConnectionRequest request, Object attempt, int responseCode, + Throwable error) { + if (!(attempt instanceof TelemetrySpan)) { + return; + } + TelemetrySpan span = (TelemetrySpan) attempt; + // Only if it is still the value this attempt set; an app's own + // traceparent, or one it replaced ours with, is left alone. + request.removeRequestHeaderIfUnchanged(TRACEPARENT, span.getTraceparent()); + if (responseCode >= 100) { + span.setAttribute("http.response.status_code", responseCode); + } + if (error != null) { + span.recordException(error); + } else if (responseCode >= 400) { + span.setError(String.valueOf(responseCode)); + } else if (responseCode < 100) { + // Neither a response nor an exception: the request was killed or + // stopped after it had started. Ended without a status, it read as + // a success in every trace backend. + span.setError("cancelled before a response"); + } + span.end(); + } + + /// Where the trace context may go. See [TelemetryConfig#propagateTo(String)] + /// for why the web is different. + private boolean shouldPropagate(String url, String host) { + if (config.propagateToAll || isSameOriginRelative(url)) { + return true; + } + if (backendOrigin != null && backendOrigin.equals(origin(url))) { + return true; + } + if (host != null) { + // Hosts the app named itself, as it named them. + for (String allowed : config.propagateTo) { + if (host.equalsIgnoreCase(allowed)) { + return true; + } + } + } + return !"HTML5".equals(Display.getInstance().getPlatformName()); + } + } + + /// The export's own request, a type of its own so the tracer can recognise it, + /// and a binary body without an intermediate String. + static final class ExportRequest extends ConnectionRequest { + /// The installation that recorded these spans. Its consent policy, not + /// whichever installation is current when the export finally runs, decides + /// whether they may be sent: after an uninstall, or a reinstall without the + /// consent flag, the static slot says nothing about THESE spans. + private final State origin; + + ExportRequest(State origin, byte[] body) { + this.origin = origin; + setRequestBody(new ByteBody(body)); + } + + /// The acknowledgement is read and DROPPED, at most 64KB of it. An OTLP + /// success body is empty or a few bytes; the inherited reader kept the whole + /// stream in memory, so a misbehaving collector or proxy answering 200 with + /// a large body could spend a phone's memory on every export. + @Override + protected void readResponse(InputStream input) throws IOException { + byte[] discard = new byte[4096]; + int total = 0; + int read; + while (total < 65536 && (read = input.read(discard)) > 0) { + total += read; + } + } + + /// None of the app's default headers. They are the app's credentials for + /// its own services; copied onto an export they reached a third-party + /// collector, and a default Content-Type relabelled the body. An export + /// carries only what the telemetry configuration names. + @Override + protected boolean shouldApplyDefaultHeaders() { + return false; + } + + /// Identity. The inherited equality compares URL and arguments, so every + /// export to one collector compared equal though each carries its own + /// spans; that made no two exports distinguishable to the queue. + @Override + public boolean equals(Object o) { + return o == this; //NOPMD CompareObjectsWithEquals + } + + @Override + public int hashCode() { + return System.identityHashCode(this); + } + + /// Stopped, and nothing sent, once consent is required and no longer + /// given. An export waits in the queue behind the app's own requests, and + /// a user who withdraws consent in that time has refused these spans too; + /// performOperationComplete asks this before it connects. + @Override + protected boolean shouldStop() { + if (super.shouldStop()) { + return true; + } + return !origin.permitted(); + } + } + + /// A request body that is already bytes. + private static final class ByteBody implements Data { + private final byte[] body; + + ByteBody(byte[] body) { + this.body = body; + } + + @Override + public void appendTo(OutputStream output) throws IOException { + output.write(body); + } + + @Override + public long getSize() { + return body.length; + } + } + + /// Whether `url` is relative to the page, `/api/orders` or `orders?id=1`, and + /// so goes to the origin that served the app. The stock + /// `ConnectionRequest.validate()` refuses such a URL, but a request that + /// overrides it can send one, and in the browser it reaches the app's own + /// backend -- the one destination that never needs a CORS allowance. Refusing + /// it, which [#host(String)] returning null did, dropped the context from the + /// call it matters most on. A scheme (`data:`, `mailto:`) or a + /// protocol-relative `//host/...` names another origin and is not relative. + private static boolean isSlash(char c) { + return c == '/' || c == '\\'; + } + + static boolean isSameOriginRelative(String url) { + if (url == null) { + return false; + } + // Read as the browser reads it (the WHATWG URL parser): leading and + // trailing C0 controls and spaces are stripped, and every tab and newline + // removed, BEFORE anything is resolved -- so " //host/x" and "\t\\\\host/x" + // are network paths to it, and were same-origin here. + StringBuilder cleaned = new StringBuilder(url.length()); + for (int i = 0; i < url.length(); i++) { + char c = url.charAt(i); + if (c != '\t' && c != '\n' && c != '\r') { + cleaned.append(c); + } + } + int start = 0; + int end = cleaned.length(); + while (start < end && cleaned.charAt(start) <= ' ') { + start++; + } + while (end > start && cleaned.charAt(end - 1) <= ' ') { + end--; + } + // Through toString(): CLDC11's StringBuilder has no substring. + url = cleaned.toString().substring(start, end); + if (url.length() == 0) { + return false; + } + // A network-path reference names another origin, and a browser's URL + // parser reads '\' as '/' for http(s): "\\host/x" and "/\\host" are + // "//host/x" to it. Refused in any mix of the two, or the trace header + // went cross-origin without a propagateTo allowance. + if (url.length() >= 2 && isSlash(url.charAt(0)) && isSlash(url.charAt(1))) { + return false; + } + for (int i = 0; i < url.length(); i++) { + char c = url.charAt(i); + if (c == ':') { + return false; + } + if (c == '/' || c == '?' || c == '#') { + return true; + } + } + return true; + } + + /// `scheme://host:port` of an http or https URL, lower case, with the + /// scheme's default port filled in, so two spellings of one origin compare + /// equal; null for anything else. + static String origin(String url) { + String host = host(url); + if (host == null) { + return null; + } + String scheme; + int defaultPort; + if (url.regionMatches(true, 0, "https://", 0, 8)) { + scheme = "https"; + defaultPort = 443; + } else if (url.regionMatches(true, 0, "http://", 0, 7)) { + scheme = "http"; + defaultPort = 80; + } else { + return null; + } + int start = scheme.length() + 3; + String authority = url.substring(start, authorityEnd(url, start)); + authority = authority.substring(authority.lastIndexOf('@') + 1); + int close = authority.lastIndexOf(']'); + int colon = authority.lastIndexOf(':'); + int port = defaultPort; + if (colon > close && colon + 1 < authority.length()) { + try { + port = Integer.parseInt(authority.substring(colon + 1)); + } catch (NumberFormatException err) { + return null; + } + } + // host() drops an IPv6 literal's brackets; an origin keeps them, or the + // port could not be told from the address. + String name = asciiLower(host); + if (name.indexOf(':') >= 0) { + name = "[" + name + "]"; + } + return scheme + "://" + name + ":" + port; + } + + private static String asciiLower(String value) { + StringBuilder out = new StringBuilder(value.length()); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + out.append(c >= 'A' && c <= 'Z' ? (char) (c + 32) : c); + } + return out.toString(); + } + + /// The host of an absolute URL, without port or userinfo; null when there is + /// none. + static String host(String url) { + if (url == null) { + return null; + } + int scheme = url.indexOf("://"); + if (scheme < 0) { + return null; + } + int start = scheme + 3; + String authority = url.substring(start, authorityEnd(url, start)); + int at = authority.lastIndexOf('@'); + if (at >= 0) { + authority = authority.substring(at + 1); + } + if (authority.startsWith("[")) { + int close = authority.indexOf(']'); + return close > 0 ? authority.substring(1, close) : authority; + } + int colon = authority.indexOf(':'); + return colon >= 0 ? authority.substring(0, colon) : authority; + } + + /// The URL as it may be recorded: no query, no fragment, no userinfo. A query + /// is where tokens and personal data travel, and a trace backend is not where + /// either belongs. + static String redact(String url) { + int cut = url.length(); + int query = url.indexOf('?'); + int fragment = url.indexOf('#'); + if (query >= 0) { + cut = query; + } + if (fragment >= 0 && fragment < cut) { + cut = fragment; + } + String out = url.substring(0, cut); + int scheme = out.indexOf("://"); + if (scheme >= 0) { + // The LAST '@' before the path, as URL parsers split it: in + // "https://alice:secret@tenant@host/x" the first '@' belongs to the + // password, and cutting there exported "tenant@" as part of url.full. + // Only '/' ends the authority here, not authorityEnd's backslash: the + // browser reads one as a slash but the other ports' URL parsers do + // not, so what it separates may be userinfo, and a redactor that + // hides too much is the safe way to be wrong. + int start = scheme + 3; + int slash = out.indexOf('/', start); + int at = out.substring(start, slash < 0 ? out.length() : slash).lastIndexOf('@'); + if (at >= 0) { + out = out.substring(0, start) + out.substring(start + at + 1); + } + } + return out; + } + + /// Where the authority that starts at `start` ends: the first '/', '\\', '?' or + /// '#'. The backslash counts because an http(s) URL parser -- the browser the + /// JavaScript port runs in, among them -- reads it as a slash, so + /// "https://evil.example\\@api.example/x" goes to evil.example. Reading the + /// backslash as part of the authority instead named api.example as the host, + /// and a trace context approved for that host went to the other one. + static int authorityEnd(String url, int start) { + for (int i = start; i < url.length(); i++) { + char c = url.charAt(i); + if (c == '/' || c == '\\' || c == '?' || c == '#') { + return i; + } + } + return url.length(); + } + + private static boolean isZero(String hex) { + for (int i = 0; i < hex.length(); i++) { + if (hex.charAt(i) != '0') { + return false; + } + } + return true; + } +} diff --git a/CodenameOne/src/com/codename1/telemetry/TelemetryConfig.java b/CodenameOne/src/com/codename1/telemetry/TelemetryConfig.java new file mode 100644 index 00000000000..312447e7a88 --- /dev/null +++ b/CodenameOne/src/com/codename1/telemetry/TelemetryConfig.java @@ -0,0 +1,606 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.telemetry; + +import java.util.ArrayList; +import java.util.List; + +/// Where [Telemetry] sends spans, and how. +/// +/// Two ways to reach a collector: +/// +/// - **Relay** (the default): the app posts OTLP/JSON to its own backend -- a +/// Codename One backend with `cn1.otel.relay=true` -- which adds the collector's +/// credentials and forwards the spans. Nothing secret ships in the app, and a +/// browser never has to reach a third-party collector, so the JavaScript port +/// needs no CORS setup on it. +/// - **Direct**: the app posts to an OTLP/HTTP collector itself. Simpler to stand +/// up, but whatever header authenticates the export is inside the app package, +/// and anything inside an app package is public. Use an ingest token scoped to +/// writing traces and nothing else. +/// +/// ```java +/// Telemetry.install(new TelemetryConfig() +/// .relay("https://api.example.com") +/// .serviceName("shop-app")); +/// ``` +public final class TelemetryConfig { + /// How spans leave the app. + public enum Mode { + /// Through the app's own backend, which forwards to the collector. + RELAY, + /// Straight to an OTLP/HTTP collector. + DIRECT + } + + Mode mode = Mode.RELAY; + String endpoint; + String serviceName; + String relayToken; + boolean protobuf = true; + double sampleRatio = 1; + int batchSize = 32; + int flushIntervalMillis = 10000; + final List headers = new ArrayList(); + final List propagateTo = new ArrayList(); + boolean propagateToAll; + boolean requireAnalyticsConsent; + + /// A copy the caller cannot reach, taken when telemetry is installed. The + /// installation read the caller's object live, while caching what it derives + /// from it (the export URL, the relay's origin) -- so reusing the config after + /// install, say `direct(...)` on one installed as a relay, sent direct-mode + /// protobuf to the old relay, which dropped every batch. EVERY field goes + /// here; a new one that is not copied is read live again. + TelemetryConfig copy() { + TelemetryConfig out = new TelemetryConfig(); + out.mode = mode; + out.endpoint = endpoint; + out.serviceName = serviceName; + out.relayToken = relayToken; + out.protobuf = protobuf; + out.sampleRatio = sampleRatio; + out.batchSize = batchSize; + out.flushIntervalMillis = flushIntervalMillis; + for (String[] header : headers) { + out.headers.add(new String[] {header[0], header[1]}); + } + out.propagateTo.addAll(propagateTo); + out.propagateToAll = propagateToAll; + out.requireAnalyticsConsent = requireAnalyticsConsent; + return out; + } + + /// Sends spans through a Codename One backend's relay. + /// + /// #### Parameters + /// + /// - `backendUrl`: the backend's base URL, such as `https://api.example.com`; + /// `/otel/v1/traces` is appended. A URL that already names a path ending in + /// `/v1/traces` is used as it is, for a relay mounted elsewhere. + /// + /// #### Returns + /// + /// this configuration + /// + /// #### Throws + /// + /// - `IllegalArgumentException`: when the URL is not http or https with a + /// host and a valid port + public TelemetryConfig relay(String backendUrl) { + this.mode = Mode.RELAY; + this.endpoint = checkedEndpoint(backendUrl); + return this; + } + + /// Sends spans straight to an OTLP/HTTP collector. + /// + /// #### Parameters + /// + /// - `collectorUrl`: the collector's base URL, `/v1/traces` appended unless + /// it is already there -- the same rule `OTEL_EXPORTER_OTLP_ENDPOINT` + /// follows everywhere + /// + /// #### Returns + /// + /// this configuration + /// + /// #### Throws + /// + /// - `IllegalArgumentException`: when the URL is not http or https with a + /// host and a valid port + public TelemetryConfig direct(String collectorUrl) { + this.mode = Mode.DIRECT; + this.endpoint = checkedEndpoint(collectorUrl); + return this; + } + + /// Refuses an endpoint no export could reach, when it is given. The build + /// checks an annotation's URL, but a configuration written in code never meets + /// that check: `direct("https://")` became `https:/v1/traces`, telemetry + /// reported itself installed, and every export -- which fails silently by + /// design -- was lost. Null or empty still means "no endpoint". + private static String checkedEndpoint(String url) { + if (url == null || url.trim().length() == 0) { + return url; + } + if (!isHttpUrl(url.trim())) { + throw new IllegalArgumentException("A telemetry endpoint must be an http or https " + + "URL with a host, such as https://collector.example:4318; it is '" + + Telemetry.redact(url.trim()) + "'"); + } + return url; + } + + /// Whether `url` is http or https with a host (a DNS name, an IPv4 address or + /// a bracketed IPv6 literal) and, if it names one, a port from 1 to 65535. + static boolean isHttpUrl(String url) { + // No fragment. HTTP never sends one, so a credential kept there + // ("#api-key=...") never reached the collector: telemetry installed and + // every export was refused, silently. + if (url.indexOf('#') >= 0) { + return false; + } + // The WHOLE URL first: no space, control or DEL anywhere. Only the + // authority was checked, so a space in the path passed and every export + // then failed at transport, silently. + for (int i = 0; i < url.length(); i++) { + char c = url.charAt(i); + if (c <= 0x20 || c == 0x7f) { + return false; + } + } + int start; + if (url.regionMatches(true, 0, "http://", 0, 7)) { + start = 7; + } else if (url.regionMatches(true, 0, "https://", 0, 8)) { + start = 8; + } else { + return false; + } + int end = url.length(); + for (int i = start; i < url.length(); i++) { + char c = url.charAt(i); + if (c == '/' || c == '?' || c == '#') { + end = i; + break; + } + } + String authority = url.substring(start, end); + int at = authority.lastIndexOf('@'); + if (at >= 0 && !validUserinfo(authority.substring(0, at))) { + return false; + } + String hostPort = authority.substring(at + 1); + String host; + String port = null; + if (hostPort.startsWith("[")) { + int close = hostPort.indexOf(']'); + if (close < 0) { + return false; + } + host = hostPort.substring(1, close); + if (!isIpv6(host)) { + return false; + } + String rest = hostPort.substring(close + 1); + if (rest.length() > 0) { + if (rest.charAt(0) != ':') { + return false; + } + port = rest.substring(1); + } + } else { + int colon = hostPort.lastIndexOf(':'); + host = colon < 0 ? hostPort : hostPort.substring(0, colon); + port = colon < 0 ? null : hostPort.substring(colon + 1); + if (!onlyChars(host, "-._~")) { + return false; + } + } + if (host.length() == 0) { + return false; + } + if (port == null || port.length() == 0) { + return true; + } + if (port.length() > 5) { + return false; + } + int value = 0; + for (int i = 0; i < port.length(); i++) { + char c = port.charAt(i); + if (c < '0' || c > '9') { + return false; + } + value = value * 10 + (c - '0'); + } + return value >= 1 && value <= 65535; + } + + /// RFC 3986 userinfo: unreserved characters, sub-delims, ':' and complete + /// percent escapes. Skipped over, a space, a control or a stray '%' in it passed + /// validation and failed only at transport, where the export fails silently. + static boolean validUserinfo(String userinfo) { + for (int i = 0; i < userinfo.length(); i++) { + char c = userinfo.charAt(i); + if (c == '%') { + if (i + 2 >= userinfo.length() || !isHex(userinfo.charAt(i + 1)) + || !isHex(userinfo.charAt(i + 2))) { + return false; + } + i += 2; + continue; + } + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') + || "-._~!$&'()*+,;=:".indexOf(c) >= 0)) { + return false; + } + } + return true; + } + + private static boolean isHex(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + } + + /// An IPv6 address by its STRUCTURE (RFC 4291 2.2), not its characters: groups + /// of one to four hex digits, at most one "::", eight groups without it and at + /// most seven with it, and optionally a dotted IPv4 tail counting as two. A + /// character check let "[:::]" through, and the transport refused every export. + static boolean isIpv6(String s) { + int n = s.length(); + if (n == 0) { + return false; + } + int groups = 0; + boolean compressed = false; + int i = 0; + if (s.startsWith("::")) { + compressed = true; + i = 2; + if (i == n) { + return true; + } + } else if (s.charAt(0) == ':') { + return false; + } + while (i < n) { + int j = i; + while (j < n && s.charAt(j) != ':') { + j++; + } + String part = s.substring(i, j); + if (part.indexOf('.') >= 0) { + if (j != n || !isIpv4(part)) { + return false; + } + groups += 2; + } else { + if (part.length() < 1 || part.length() > 4) { + return false; + } + for (int k = 0; k < part.length(); k++) { + char c = part.charAt(k); + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) { + return false; + } + } + groups++; + } + if (j == n) { + break; + } + if (j + 1 < n && s.charAt(j + 1) == ':') { + if (compressed) { + return false; + } + compressed = true; + i = j + 2; + } else { + i = j + 1; + if (i == n) { + return false; + } + } + } + return compressed ? groups <= 7 : groups == 8; + } + + /// Four decimal parts, each 0 to 255. + static boolean isIpv4(String s) { + int parts = 0; + int i = 0; + while (i <= s.length()) { + int j = s.indexOf('.', i); + if (j < 0) { + j = s.length(); + } + String part = s.substring(i, j); + if (part.length() < 1 || part.length() > 3) { + return false; + } + int value = 0; + for (int k = 0; k < part.length(); k++) { + char c = part.charAt(k); + if (c < '0' || c > '9') { + return false; + } + value = value * 10 + (c - '0'); + } + if (value > 255) { + return false; + } + parts++; + i = j + 1; + } + return parts == 4; + } + + /// The headers the exporter owns: its body's type and framing, and the + /// destination, which come from what it sends and where. + static boolean isExporterOwned(String name) { + return "Content-Type".equalsIgnoreCase(name) || "Content-Length".equalsIgnoreCase(name) + || "Host".equalsIgnoreCase(name) || "Transfer-Encoding".equalsIgnoreCase(name); + } + + private static boolean onlyChars(String value, String extra) { + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') + || extra.indexOf(c) >= 0)) { + return false; + } + } + return true; + } + + /// The `service.name` the app's spans are reported under. Defaults to the + /// app's name. + /// + /// #### Returns + /// + /// this configuration + public TelemetryConfig serviceName(String name) { + this.serviceName = name; + return this; + } + + /// A header sent with every direct export -- the collector's credential. + /// Ignored in relay mode, where the backend holds the credential. + /// + /// #### Returns + /// + /// this configuration + /// + /// #### Throws + /// + /// - `IllegalArgumentException`: when the name is not an HTTP token or the + /// value holds a control character + public TelemetryConfig header(String name, String value) { + if (name != null && value != null) { + checkHeader(name, value); + headers.add(new String[] {name, value}); + } + return this; + } + + /// The rules the annotation processor applies to a header, for one written in + /// code, which never meets it. A bad header is refused only when an export + /// runs -- after its batch has left the buffer, and silently, by design -- so + /// accepting it here lost every batch. The message names the header, never + /// the value, which is usually a credential. + private static void checkHeader(String name, String value) { + if (name.length() == 0) { + throw new IllegalArgumentException("A telemetry header needs a name"); + } + if (isExporterOwned(name)) { + // The exporter sets these from what it sends. A Content-Type given + // here replaced the media type -- addRequestHeader treats it as + // setContentType -- so protobuf went out labelled JSON and every batch + // was refused, after it had left the buffer. + throw new IllegalArgumentException(name + " is set by the exporter from what it " + + "sends and cannot be configured"); + } + for (int i = 0; i < name.length(); i++) { + char c = name.charAt(i); + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') + || "!#$%&'*+-.^_`|~".indexOf(c) >= 0)) { + throw new IllegalArgumentException("'" + name + "' is not a valid HTTP header name"); + } + } + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if ((c < 0x20 && c != '\t') || c == 0x7f) { + throw new IllegalArgumentException("The value of telemetry header " + name + + " holds a control character, which no HTTP header may carry"); + } + } + } + + /// The shared secret a backend's relay may require + /// (`cn1.otel.relay.token`). It keeps casual traffic off the relay; it is not + /// a credential, since it ships in the app. + /// + /// #### Returns + /// + /// this configuration + /// + /// #### Throws + /// + /// - `IllegalArgumentException`: when the token holds a control character, or + /// begins or ends with whitespace + public TelemetryConfig relayToken(String token) { + if (token != null) { + // Sent as a header, so held to the same rules -- and compared exactly, + // while the backend trims the spaces and tabs around a header value, so + // a token with them could never match. + checkHeader("X-CN1-Telemetry-Token", token); + int last = token.length() - 1; + if (last >= 0 && (token.charAt(0) == ' ' || token.charAt(0) == '\t' + || token.charAt(last) == ' ' || token.charAt(last) == '\t')) { + throw new IllegalArgumentException("The relay token begins or ends with " + + "whitespace, which a header cannot carry"); + } + } + this.relayToken = token; + return this; + } + + /// Whether direct exports use binary protobuf (the default) or JSON. Some + /// collectors accept only protobuf. The relay always receives JSON, and + /// re-encodes it for the collector as the backend is configured to. + /// + /// #### Returns + /// + /// this configuration + public TelemetryConfig protobuf(boolean protobuf) { + this.protobuf = protobuf; + return this; + } + + /// The share of NEW traces recorded, from 0 to 1. A request made inside a + /// span follows that span's decision, and the backend follows the app's. + /// + /// #### Returns + /// + /// this configuration + /// + /// #### Throws + /// + /// - `IllegalArgumentException`: for NaN + public TelemetryConfig sampleRatio(double ratio) { + // NaN fails both comparisons below, so it was stored as it was, and the + // sampler then declined every trace while telemetry reported itself on. + // A range can be clamped into; NaN has no nearest value, so it is refused + // -- as the annotation processor refuses it. + if (Double.isNaN(ratio)) { + throw new IllegalArgumentException("The telemetry sample ratio is NaN; " + + "give a number from 0 to 1"); + } + this.sampleRatio = ratio < 0 ? 0 : ratio > 1 ? 1 : ratio; + return this; + } + + /// How many ended spans are buffered before an export. Defaults to 32. + /// + /// #### Returns + /// + /// this configuration + public TelemetryConfig batchSize(int size) { + this.batchSize = size < 1 ? 1 : size; + return this; + } + + /// How often buffered spans are exported even when the batch is not full. + /// Defaults to ten seconds. + /// + /// #### Returns + /// + /// this configuration + public TelemetryConfig flushIntervalMillis(int millis) { + this.flushIntervalMillis = millis < 1000 ? 1000 : millis; + return this; + } + + /// Sends the W3C trace context to requests for this host as well. + /// + /// By default the context goes to the relay's host -- the app's own backend -- + /// and, on every platform except the web, to every host. The web is the + /// exception because `traceparent` is not a CORS-safelisted header: sending it + /// to a server that does not allow it turns a working cross-origin request + /// into a failed preflight. Name the hosts that do allow it here. + /// + /// #### Returns + /// + /// this configuration + public TelemetryConfig propagateTo(String host) { + if (host != null && host.length() > 0) { + propagateTo.add(host); + } + return this; + } + + /// Sends the trace context to every host, on every platform, the web + /// included. For an app whose every request goes to servers that allow the + /// header. + /// + /// #### Returns + /// + /// this configuration + public TelemetryConfig propagateToAllHosts() { + this.propagateToAll = true; + return this; + } + + /// Records and propagates traces only while the user has granted ANALYTICS + /// consent through `com.codename1.analytics.Analytics`, the same consent the + /// analytics providers honour. Off by default: whether trace data needs consent + /// is the app's decision, and depends on what it records and where it ships. + /// + /// With it on and no consent, requests are sent exactly as they would be + /// without telemetry -- no span, and no `traceparent` header -- and spans + /// already buffered are dropped rather than exported. Before the user answers, + /// the analytics `ConsentMode` decides: `OPT_IN` (the default) means no. + /// + /// #### Returns + /// + /// this configuration + public TelemetryConfig requireAnalyticsConsent(boolean require) { + this.requireAnalyticsConsent = require; + return this; + } + + /// The URL spans are posted to, or null when no endpoint was given. + String exportUrl() { + if (endpoint == null || endpoint.length() == 0) { + return null; + } + String url = endpoint.trim(); + // The path is what gets the suffix; a query or fragment -- where a + // collector's api-key often travels -- is set aside and put back after it. + // Appending to the whole string put the path inside the credential and + // sent the export to the base path. + int cut = url.length(); + int query = url.indexOf('?'); + int fragment = url.indexOf('#'); + if (query >= 0) { + cut = query; + } + if (fragment >= 0 && fragment < cut) { + cut = fragment; + } + String base = url.substring(0, cut); + String suffix = url.substring(cut); + // Trailing slashes first: ".../v1/traces/" is the full URL too, and testing + // before stripping appended the path a second time -- a route no collector + // serves, and the export fails silently by design. + while (base.endsWith("/")) { + base = base.substring(0, base.length() - 1); + } + if (base.endsWith("/v1/traces")) { + return base + suffix; + } + return base + (mode == Mode.RELAY ? "/otel/v1/traces" : "/v1/traces") + suffix; + } +} diff --git a/CodenameOne/src/com/codename1/telemetry/TelemetrySpan.java b/CodenameOne/src/com/codename1/telemetry/TelemetrySpan.java new file mode 100644 index 00000000000..9e904ef74d5 --- /dev/null +++ b/CodenameOne/src/com/codename1/telemetry/TelemetrySpan.java @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.telemetry; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/// One timed operation in a distributed trace. +/// +/// Network requests become spans by themselves once [Telemetry] is installed; an +/// app creates its own with [Telemetry#startSpan(String)] or +/// [Telemetry#run(String, Runnable)] to time something the user did -- a tap, a +/// screen load -- so the requests it caused are grouped under it. +/// +/// A span the sampler declined is not recorded, but it still carries its trace +/// context, and propagates the decision: the backend it calls agrees not to record +/// either, so a trace is whole or absent rather than missing its middle. +/// +/// Not thread safe. A span belongs to the thread doing the work it times, and is +/// handed to the exporter only when it ends. +public final class TelemetrySpan { + /// An operation inside the app. + public static final int KIND_INTERNAL = 1; + /// A request the app makes to something else. + public static final int KIND_CLIENT = 3; + + /// OpenTelemetry's default limits. Past them an attribute or event is counted, + /// not stored. + static final int MAX_ATTRIBUTES = 128; + static final int MAX_EVENTS = 32; + static final int MAX_VALUE_LENGTH = 4096; + /// Past this an attribute KEY is dropped; see put. + static final int MAX_KEY_LENGTH = 256; + + final String traceId; + final String spanId; + final String parentSpanId; + final boolean sampled; + final int kind; + String name; + final long startEpochNanos; + long endEpochNanos; + /// The clock this span reads: an epoch time and a monotonic reading taken at the + /// same moment, inherited from a local parent so every span of one trace sits on + /// one timeline. A device's wall clock is corrected while the app runs -- + /// network time, the user, a time zone change -- and a child that read it for + /// itself could start before its parent, or long after the parent had ended. + final long anchorEpochNanos; + final long anchorNano; + final Map attributes; + int droppedAttributes; + /// Each an Object[] {Long time, String name, Map attributes}. + final List events; + int droppedEvents; + /// 0 unset, 2 error: OTLP's own status codes. + int statusCode; + String statusMessage; + private boolean ended; + private final Telemetry.State owner; + + TelemetrySpan(Telemetry.State owner, String name, int kind, String traceId, String spanId, + String parentSpanId, boolean sampled) { + this(owner, name, kind, traceId, spanId, parentSpanId, sampled, null); + } + + TelemetrySpan(Telemetry.State owner, String name, int kind, String traceId, String spanId, + String parentSpanId, boolean sampled, TelemetrySpan localParent) { + this.owner = owner; + // Bounded like every other string a span keeps: a caller-derived name + // could otherwise be any size, and the buffer is bounded by span COUNT. + this.name = name == null ? "" : bound(name); + this.kind = kind; + this.traceId = traceId; + this.spanId = spanId; + this.parentSpanId = parentSpanId; + this.sampled = sampled; + if (localParent != null) { + this.anchorEpochNanos = localParent.anchorEpochNanos; + this.anchorNano = localParent.anchorNano; + } else { + this.anchorEpochNanos = System.currentTimeMillis() * 1000000L; + this.anchorNano = System.nanoTime(); + } + this.startEpochNanos = now(); + this.attributes = sampled ? new LinkedHashMap() : null; + this.events = sampled ? new ArrayList() : null; + } + + /// Adds or replaces a string attribute. Null values are ignored. + /// + /// #### Returns + /// + /// this span + public TelemetrySpan setAttribute(String key, String value) { + if (value != null && value.length() > MAX_VALUE_LENGTH) { + return put(key, bound(value)); + } + return put(key, value); + } + + /// Adds or replaces an integer attribute. + /// + /// #### Returns + /// + /// this span + public TelemetrySpan setAttribute(String key, long value) { + return put(key, Long.valueOf(value)); + } + + /// Adds or replaces a floating point attribute. NaN and the infinities, which + /// JSON cannot spell, are recorded as text. + /// + /// #### Returns + /// + /// this span + public TelemetrySpan setAttribute(String key, double value) { + if (Double.isNaN(value) || Double.isInfinite(value)) { + return put(key, String.valueOf(value)); + } + return put(key, Double.valueOf(value)); + } + + /// Adds or replaces a flag attribute. + /// + /// #### Returns + /// + /// this span + public TelemetrySpan setAttribute(String key, boolean value) { + return put(key, value ? Boolean.TRUE : Boolean.FALSE); + } + + private TelemetrySpan put(String key, Object value) { + if (!sampled || ended || key == null || value == null) { + return this; + } + // An oversized KEY is dropped rather than cut. Values are truncated because a + // prefix of a value is still that value; a prefix of a key is a different + // attribute, and two long keys sharing one would silently overwrite each other. + // Left unbounded, one key read from a request could carry the whole export + // batch past the collector's size limit and lose every span in it. + if (key.length() > MAX_KEY_LENGTH) { + droppedAttributes++; + return this; + } + if (!attributes.containsKey(key) && attributes.size() >= MAX_ATTRIBUTES) { + droppedAttributes++; + return this; + } + attributes.put(key, value); + return this; + } + + /// Records a failure as an "exception" event and marks the span failed. The + /// message is kept; the stack trace is not. + /// + /// #### Returns + /// + /// this span + public TelemetrySpan recordException(Throwable error) { + if (!sampled || ended || error == null) { + return this; + } + String type = error.getClass().getName(); + String message = error.getMessage(); + statusCode = 2; + // Bounded like the event attribute below: an exception message can be any + // size, and the export queue is bounded by span count, not bytes. + statusMessage = bound(message == null ? type : message); + if (events.size() >= MAX_EVENTS) { + droppedEvents++; + return this; + } + Map attrs = new LinkedHashMap(); + attrs.put("exception.type", type); + if (message != null) { + attrs.put("exception.message", message.length() > MAX_VALUE_LENGTH + ? bound(message) : message); + } + Object[] event = {Long.valueOf(now()), "exception", attrs}; + events.add(event); + return this; + } + + /// Marks the span failed, with a short description. + /// + /// #### Returns + /// + /// this span + public TelemetrySpan setError(String description) { + if (sampled && !ended) { + statusCode = 2; + statusMessage = description == null ? null : bound(description); + } + return this; + } + + /// Renames the span. + /// + /// #### Returns + /// + /// this span + public TelemetrySpan updateName(String newName) { + if (!ended && newName != null) { + name = bound(newName); + } + return this; + } + + /// The span's name. + public String getName() { + return name; + } + + /// Whether this span is recorded. Attributes set on one that is not go nowhere, + /// so an attribute that is expensive to compute can be skipped. + public boolean isRecording() { + return sampled && !ended; + } + + /// The 32 hex digit trace id. + public String getTraceId() { + return traceId; + } + + /// The 16 hex digit span id. + public String getSpanId() { + return spanId; + } + + /// This span as a W3C `traceparent` header value, for a transport the + /// framework does not instrument itself -- a WebSocket message, a push token + /// registration. + /// + /// Null for the span [Telemetry#startSpan(String)] hands out when there is + /// no trace to join: telemetry is not installed, consent is required and not + /// given, or the platform cannot make ids. Its ids are all zeros, which W3C + /// Trace Context defines as invalid, so a header built from them would be + /// refused or misread downstream. An unsampled span is different: it is a real + /// trace whose decision must travel, and it answers with its `-00` flags. + /// + /// #### Returns + /// + /// the header value, or null when there is no trace + public String getTraceparent() { + if (owner == null) { + return null; + } + return "00-" + traceId + "-" + spanId + (sampled ? "-01" : "-00"); + } + + /// Whether `state` recorded this span. + boolean isOwnedBy(Telemetry.State state) { + return owner == state; //NOPMD CompareObjectsWithEquals + } + + /// Ends the span. Only the first call counts. + public void end() { + if (ended) { + return; + } + ended = true; + endEpochNanos = now(); + if (sampled && owner != null) { + owner.ended(this); + } + } + + /// Epoch nanoseconds on this trace's clock: the anchor plus the monotonic time + /// since it. Never earlier than the anchor, whatever the platform's clock does. + private long now() { + long elapsed = System.nanoTime() - anchorNano; + return anchorEpochNanos + (elapsed < 0 ? 0 : elapsed); + } + + /// At most MAX_VALUE_LENGTH chars, cut on a code point boundary. A cut through + /// a surrogate pair kept a lone high surrogate, which UTF-8 encoding then + /// replaced, so the exported value was not the one the app recorded. + static String bound(String value) { + if (value.length() <= MAX_VALUE_LENGTH) { + return value; + } + int end = MAX_VALUE_LENGTH; + if (Character.isHighSurrogate(value.charAt(end - 1))) { + end--; + } + return value.substring(0, end); + } +} diff --git a/CodenameOne/src/com/codename1/telemetry/package-info.java b/CodenameOne/src/com/codename1/telemetry/package-info.java new file mode 100644 index 00000000000..2a7f71ab327 --- /dev/null +++ b/CodenameOne/src/com/codename1/telemetry/package-info.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +/// OpenTelemetry tracing for the app, exported over OTLP/HTTP. +/// +/// Installed by the build from `@OpenTelemetry` on the main class, or by hand with +/// {@link com.codename1.telemetry.Telemetry#install(com.codename1.telemetry.TelemetryConfig)}. +/// Every {@link com.codename1.io.ConnectionRequest} then becomes a span and sends the +/// W3C `traceparent` header, so a Codename One backend -- or any service that speaks +/// W3C Trace Context -- continues the app's trace instead of starting its own. +/// +/// Spans leave the app either through the app's own backend, which relays them to +/// the collector with credentials the app never holds, or directly to a collector; +/// see {@link com.codename1.telemetry.TelemetryConfig}. +package com.codename1.telemetry; diff --git a/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEPort.java b/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEPort.java index c9bb64e1617..9056edff3c7 100644 --- a/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEPort.java +++ b/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEPort.java @@ -11721,7 +11721,8 @@ public void postInit() { "cn1app.ProtoBootstrap", "cn1app.GrpcClientBootstrap", "cn1app.GraphQLClientBootstrap", - "cn1app.IntentBootstrap"}) { + "cn1app.IntentBootstrap", + "cn1app.TelemetryBootstrap"}) { try { Class.forName(bootstrap).newInstance(); } catch (ClassNotFoundException ignored) { diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/AppTracingSnippets.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/AppTracingSnippets.java new file mode 100644 index 00000000000..20381e51aa6 --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/AppTracingSnippets.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend; + +import com.codename1.annotations.OpenTelemetry; +import com.codename1.system.Lifecycle; +import com.codename1.telemetry.Telemetry; +import com.codename1.telemetry.TelemetryConfig; + +/// The app tracing chapter's examples, compiled so they cannot drift. They live in +/// this module, beside the server's, because it compiles against the core and runs +/// no annotation processing: in an app `@OpenTelemetry` is a switch the build acts +/// on, and two of them in one app are an error. +public final class AppTracingSnippets { + + private AppTracingSnippets() { + } + + public static final class Relay { +// tag::app-otel-relay[] +@OpenTelemetry(relay = "https://api.example.com", + serviceName = "shop-app") +public static class ShopApp extends Lifecycle { +} +// end::app-otel-relay[] + } + + public static final class Direct { +// tag::app-otel-direct[] +@OpenTelemetry( + endpoint = "https://abc12345.live.dynatrace.com/api/v2/otlp", + headers = "Authorization: Api-Token dt0c01.XXXX") +public static class ShopApp extends Lifecycle { +} +// end::app-otel-direct[] + } + + /// Stands in for whatever the app's checkout does. + public static final class Cart { + public void submit() { + } + } + + public static void timeAnAction(Cart cart) { +// tag::app-otel-run[] +Telemetry.run("checkout", () -> cart.submit()); +// end::app-otel-run[] + } + + public static void installInCode() { +// tag::app-otel-install[] +Telemetry.install(new TelemetryConfig() + .relay("https://api.example.com") + .serviceName("shop-app") + .sampleRatio(0.2)); +// end::app-otel-install[] + } +} diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/TracingSnippets.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/TracingSnippets.java new file mode 100644 index 00000000000..fad6dba2be2 --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/TracingSnippets.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend; + +import com.codename1.backend.Config; +import com.codename1.backend.Tracing; +import com.codename1.backend.annotations.GetMapping; +import com.codename1.backend.annotations.OpenTelemetry; +import com.codename1.backend.annotations.PathVariable; +import com.codename1.backend.annotations.RestController; +import com.codename1.backend.otel.OtlpTracer; + +/// The Backend chapter's tracing examples, compiled so they cannot drift. This +/// module runs no annotation processing, which is what lets `@OpenTelemetry` appear +/// here as an example: in a server module it is a switch the build acts on. +public final class TracingSnippets { + + private TracingSnippets() { + } + + public static final class Server { +// tag::backend-otel-annotation[] +@OpenTelemetry(serviceName = "notes") +@RestController +public static class NotesController { + @GetMapping("/notes/{id}") + public String note(@PathVariable("id") String id) { + return id; + } +} +// end::backend-otel-annotation[] + } + + public static void installByHand() throws Exception { +// tag::backend-otel-install[] +Tracing.install(OtlpTracer.open(Config.load(), "notes")); +// end::backend-otel-install[] + } + +} diff --git a/docs/developer-guide/Backend.asciidoc b/docs/developer-guide/Backend.asciidoc index 6381fe7567f..e211f5dc05c 100644 --- a/docs/developer-guide/Backend.asciidoc +++ b/docs/developer-guide/Backend.asciidoc @@ -606,6 +606,112 @@ Sections 12 and 13 are excluded while permessage-deflate is unimplemented. They' excluded rather than tolerated: accepting their result as a pass would also accept it for a case that used to work. +=== Tracing with OpenTelemetry + +A server can report every request it serves to any OpenTelemetry collector, and +the reporting needs no code in the handlers. Turn it on at build time with the +annotation, on any class in the backend module: + +[source,java] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/TracingSnippets.java[tag=backend-otel-annotation,indent=0] +---- + +Without touching the source, set it in `application.properties` instead: + +---- +cn1.otel.enabled=true +---- + +Either one makes the generated entry point install a tracer. From then on: + +* every request is a server span, named after the route that matched + (`GET /notes/{id}`), with its status, method and path; +* every outbound `Web` call is a client span, and sends the W3C `traceparent` + and `tracestate` headers so the service it reaches joins the same trace; +* every `Database` statement is a client span carrying the SQL as the code wrote + it, placeholders and all -- the bound values are never recorded; +* an incoming `traceparent` makes the request part of the caller's trace, which is + how a Codename One app's spans connect to the backend's own (see + <>). + +A WebSocket connection isn't a span. It can stay open for hours and carry any +number of messages, so it has no single start and end to time, and its upgrade +request is answered before tracing begins. + +Without the annotation or the property, nothing refers to the tracer and the +translator leaves it out of the binary. + +Spans go out over OTLP/HTTP, as binary protobuf by default, batched on a +thread of their own so a slow collector never holds up a request. A queue that +fills because the collector is down drops spans and counts them, and +`HttpServer.getMetrics()` reports those counts. The settings are the standard +OpenTelemetry environment variables, so a deployment configures this server the +same way it configures everything else it runs: + +[cols="2,2,3"] +|=== +| Variable | Property | Meaning + +| `OTEL_EXPORTER_OTLP_ENDPOINT` | `cn1.otel.endpoint` | The collector's base URL; `/v1/traces` is appended. Defaults to `http://localhost:4318`. +| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | `cn1.otel.traces.endpoint` | The full URL, used as it is. +| `OTEL_EXPORTER_OTLP_HEADERS` | `cn1.otel.headers` | `name=value` pairs, comma separated, with each value URL-encoded. +| `OTEL_EXPORTER_OTLP_PROTOCOL` | `cn1.otel.protocol` | `http/protobuf` or `http/json`. gRPC isn't supported. +| `OTEL_SERVICE_NAME` | `cn1.otel.service.name` | Overrides the annotation's `serviceName`. +| `OTEL_RESOURCE_ATTRIBUTES` | `cn1.otel.resource.attributes` | Extra resource attributes, `key=value` pairs. +| `OTEL_TRACES_SAMPLER` | `cn1.otel.sampler` | `parentbased_always_on` by default; also `always_on`, `always_off`, `traceidratio` and the other `parentbased_` forms. +| `OTEL_TRACES_SAMPLER_ARG` | `cn1.otel.sampler.arg` | The ratio for the ratio samplers. +| `OTEL_SDK_DISABLED` | `cn1.otel.disabled` | `true` turns tracing off at start-up without a rebuild. +|=== + +`cn1.otel.attributes.exclude` names attributes never to record, for example +`db.query.text` in a code base that builds SQL by concatenating values. + +Sending spans to Dynatrace, which accepts OTLP over HTTP as protobuf, is a +matter of pointing the endpoint at the environment's OTLP API and passing an +ingest token: + +---- +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=https://abc12345.live.dynatrace.com/api/v2/otlp/v1/traces +OTEL_EXPORTER_OTLP_HEADERS=Authorization=Api-Token%20dt0c01.XXXX +---- + +A server started without the builder -- one that calls `HttpServer.start` or runs +the Lambda loop itself -- installs the tracer in one line: + +[source,java] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/TracingSnippets.java[tag=backend-otel-install,indent=0] +---- + +Under Lambda, the invocation continues the trace the host passes in its X-Ray +header, and the loop waits for the export before it polls again, because the host +freezes the process between invocations. + +`Tracing.current()` returns the request's span, for adding an attribute of the +application's own, and `Tracing.inSpan` times a block of work as a child span. + +==== Relaying the app's spans + +A mobile app shouldn't carry the collector's credentials: anything inside an app +package can be read by whoever installs it. With `cn1.otel.relay=true` the backend +accepts the app's spans at `/otel/v1/traces`, adds its own credentials and +forwards them to the same collector: + +---- +cn1.otel.relay=true +# Optional: a shared secret the app sends in X-CN1-Telemetry-Token. +cn1.otel.relay.token=${RELAY_TOKEN} +# Only for a web app served from another origin. +cn1.otel.relay.corsOrigin=https://app.example.com +---- + +The relay takes OTLP/JSON, rebuilds it against the OTLP schema -- a field the +schema doesn't name is dropped and a malformed id is refused -- and queues it for +export in whatever protocol the server exports with. It answers as soon as the +spans are queued, and answers 503 when the queue is full so the app backs off. +`cn1.otel.relay.maxBytes` and `cn1.otel.relay.maxSpans` bound a single export. + === What it costs Same handler, three ways, plus Go for an outside reference. Two pinned cores, 64 diff --git a/docs/developer-guide/OpenTelemetry-Tracing.asciidoc b/docs/developer-guide/OpenTelemetry-Tracing.asciidoc new file mode 100644 index 00000000000..99d75788d38 --- /dev/null +++ b/docs/developer-guide/OpenTelemetry-Tracing.asciidoc @@ -0,0 +1,109 @@ +[[opentelemetry-tracing]] +== Distributed tracing with OpenTelemetry + +OpenTelemetry tracing follows one user action from the app through every service +it touches. When the user taps the checkout button, the trace shows the tap, the requests +the app made, what the backend did with each one and which database query was +slow, as one timeline. Codename One produces these traces in the standard OTLP +format, so any OpenTelemetry collector or tracing backend can display them. + +=== Turning it on + +Put `@OpenTelemetry` on the main class: + +[source,java] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/AppTracingSnippets.java[tag=app-otel-relay,indent=0] +---- + +The build generates a bootstrap that installs `com.codename1.telemetry.Telemetry` +before the app starts. From then on every `ConnectionRequest` is a span, and so +is every call through a REST, gRPC-Web or GraphQL client the build generates, +because they all go through `ConnectionRequest`. Each request also sends the W3C +`traceparent` header, so a server that understands trace context continues the +app's trace instead of starting a new one. The Codename One backend understands +it (see the backend chapter), and so does any service instrumented with +OpenTelemetry. + +An app without the annotation doesn't reference the telemetry classes, so they +aren't part of its build. + +=== Where the spans go + +There are two ways to reach a collector, and the annotation takes one of them. + +`relay` sends the spans to the app's own Codename One backend, started with +`cn1.otel.relay=true`. The backend adds the collector's credentials and forwards +the spans. This is the recommended setup: nothing secret ships in the app, and a +web build never has to reach a collector on another origin. + +`endpoint` sends the spans straight to an OTLP/HTTP collector: + +[source,java] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/AppTracingSnippets.java[tag=app-otel-direct,indent=0] +---- + +The header is compiled into the app, and anyone who installs the app can read +it. Use a token that can ingest traces and do nothing else. + +Direct exports are binary protobuf by default, which every collector accepts and +some, Dynatrace among them, require. Set `protobuf = false` for JSON. Exports +to a relay are always JSON, and the backend re-encodes them for its collector. + +=== Timing what the user did + +A request made on its own is a trace of its own. To group the requests that one +user action causes, run the action inside a span: + +[source,java] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/AppTracingSnippets.java[tag=app-otel-run,indent=0] +---- + +Requests queued while `checkout` is running become its children, and so does the +backend work they cause. `Telemetry.startSpan` returns a span you end yourself, +for work that starts in one place and finishes in another. + +=== Sampling and privacy + +`sampleRatio` records a fraction of new traces, from 0 to 1. A trace the app +decides not to record still sends `traceparent`, flagged as not sampled, and the +backend's default sampler follows the app's decision. Each trace is therefore +recorded completely or not at all. + +A span records the request's method, host, path, and status. It never records the +query string, request bodies or response bodies. + +Whether trace data needs the user's consent depends on what the app records and +where it sends it, so the app decides. With `requireAnalyticsConsent = true` on the +annotation, or `requireAnalyticsConsent(true)` on `TelemetryConfig`, tracing runs +only while the user has granted analytics consent through the `Analytics` API (see +<>). Without that consent, requests are sent exactly as they would be +with telemetry off: no span and no `traceparent` header. Spans that were buffered +before the user withdrew consent are discarded, not exported. Until the user +makes a choice, the analytics consent mode decides, and the default `OPT_IN` mode +means no tracing. + +=== The web build + +`traceparent` isn't one of the request headers a browser sends across origins +without asking. When a web app adds it to a request for another origin, the +browser first sends a CORS preflight, and a server that doesn't allow the header +fails the request. The web build therefore sends trace context only to the relay's +host and to the hosts listed in `propagateTo`. The other platforms send it to +every host. + +=== Configuring it in code + +The annotation is a shortcut for this call, which an app can also make itself, +for example to choose the endpoint at run time: + +[source,java] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/AppTracingSnippets.java[tag=app-otel-install,indent=0] +---- + +Spans are exported in batches, when 32 have ended or every ten seconds. +Call `Telemetry.flush()` when the app is paused, because spans still in memory are +lost if the process is reclaimed. diff --git a/docs/developer-guide/developer-guide.asciidoc b/docs/developer-guide/developer-guide.asciidoc index 55ba0729a3d..c9ecbac11b1 100644 --- a/docs/developer-guide/developer-guide.asciidoc +++ b/docs/developer-guide/developer-guide.asciidoc @@ -193,6 +193,8 @@ include::Advertising.asciidoc[] include::Analytics.asciidoc[] +include::OpenTelemetry-Tracing.asciidoc[] + include::App-Review.asciidoc[] include::Crash-Protection.asciidoc[] diff --git a/docs/developer-guide/languagetool-accept.txt b/docs/developer-guide/languagetool-accept.txt index ef6b8579644..9179b5b5598 100644 --- a/docs/developer-guide/languagetool-accept.txt +++ b/docs/developer-guide/languagetool-accept.txt @@ -575,6 +575,8 @@ transcoder dedup # Analytics chapter: analytics vendors / APIs and privacy-law acronyms. Matomo +# Observability backend named in the tracing chapters (OTLP ingest example). +Dynatrace GA4 CCPA SPI diff --git a/maven/backend/pom.xml b/maven/backend/pom.xml index 521c4f75bbb..8dcae0ad2ed 100644 --- a/maven/backend/pom.xml +++ b/maven/backend/pom.xml @@ -68,6 +68,26 @@ ${junit.jupiter.version} test + + + io.opentelemetry.proto + opentelemetry-proto + 1.3.2-alpha + test + + + com.google.protobuf + protobuf-java + 3.25.5 + test + diff --git a/maven/backend/src/test/java/com/codename1/backend/LambdaTracingTest.java b/maven/backend/src/test/java/com/codename1/backend/LambdaTracingTest.java new file mode 100644 index 00000000000..3e7ca4274b3 --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/LambdaTracingTest.java @@ -0,0 +1,346 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** The Lambda loop's span, and the span a failing tracer leaves behind. */ +class LambdaTracingTest { + + @AfterEach + void uninstall() { + Tracing.install(null); + } + + @Test + @DisplayName("an invocation whose result the runtime API refuses ends as a failure") + void refusedDeliveryIsRecorded() throws Exception { + HttpServer api = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + api.createContext("/2018-06-01/runtime/invocation/next", (HttpExchange ex) -> { + byte[] body = "{}".getBytes("UTF-8"); + ex.getResponseHeaders().add("Lambda-Runtime-Aws-Request-Id", "req-1"); + ex.sendResponseHeaders(200, body.length); + ex.getResponseBody().write(body); + ex.close(); + }); + api.createContext("/2018-06-01/runtime/invocation/req-1/response", (HttpExchange ex) -> { + drain(ex); + ex.sendResponseHeaders(413, -1); + ex.close(); + }); + api.createContext("/2018-06-01/runtime/invocation/req-1/error", (HttpExchange ex) -> { + drain(ex); + ex.sendResponseHeaders(202, -1); + ex.close(); + }); + api.start(); + Recorder recorder = new Recorder(); + Tracing.install(recorder); + try { + LambdaRuntime.pumpOnce(new Handler() { + public String handle(String event, String requestId) { + return "{\"ok\":true}"; + } + }, "127.0.0.1", api.getAddress().getPort()); + } finally { + api.stop(0); + } + assertEquals(1, recorder.spans.size()); + RecordedSpan span = (RecordedSpan)recorder.spans.get(0); + assertTrue(span.ended, "the invocation span was never ended"); + assertTrue(span.error != null && span.error.indexOf("413") >= 0, + "the lost result must be on the span, not the handler's success: " + span.error); + } + + @Test + @DisplayName("a failed invocation's span stays open through the error report, and says it was lost") + void anUndeliveredErrorReportIsRecorded() throws Exception { + final Recorder recorder = new Recorder(); + final boolean[] endedBeforeReport = new boolean[1]; + HttpServer api = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + api.createContext("/2018-06-01/runtime/invocation/next", (HttpExchange ex) -> { + byte[] body = "{}".getBytes("UTF-8"); + ex.getResponseHeaders().add("Lambda-Runtime-Aws-Request-Id", "req-2"); + ex.sendResponseHeaders(200, body.length); + ex.getResponseBody().write(body); + ex.close(); + }); + api.createContext("/2018-06-01/runtime/invocation/req-2/error", (HttpExchange ex) -> { + drain(ex); + endedBeforeReport[0] = ((RecordedSpan)recorder.spans.get(0)).ended; + ex.sendResponseHeaders(500, -1); + ex.close(); + }); + api.start(); + Tracing.install(recorder); + boolean keepPolling; + try { + keepPolling = LambdaRuntime.pumpOnce(new Handler() { + public String handle(String event, String requestId) { + throw new IllegalStateException("handler failed"); + } + }, "127.0.0.1", api.getAddress().getPort()); + } finally { + api.stop(0); + } + assertFalse(keepPolling, "an unreported invocation must stop the loop"); + RecordedSpan span = (RecordedSpan)recorder.spans.get(0); + assertFalse(endedBeforeReport[0], "the span ended before the error report was sent"); + assertTrue(span.ended); + assertEquals(2, span.errors.size(), String.valueOf(span.errors)); + assertEquals("handler failed", span.errors.get(0)); + assertTrue(String.valueOf(span.errors.get(1)).contains("did not accept the error report"), + String.valueOf(span.errors)); + } + + @Test + @DisplayName("a failed start-up whose tracer was replaced meanwhile retires the one it displaced") + void aRollBackAfterAnotherInstallRetiresWhatItDisplaced() { + Recorder x = new Recorder(); + Recorder failing = new Recorder(); + Recorder other = new Recorder(); + Tracing.install(x); + // Start-up A swaps X out, start-up B swaps A out and completes, then A fails. + Tracing.Swap a = Tracing.swap(failing); + Tracing.commit(Tracing.swap(other)); + Tracing.rollBack(a); + assertTrue(Tracing.getTracer() == other, "the later install was undone"); + assertEquals(1, x.shutdowns, "the tracer the failed start-up displaced leaked"); + assertEquals(1, failing.shutdowns, "retired once, by the start-up that replaced it"); + } + + @Test + @DisplayName("overlapping failed start-ups restore the original tracer and stop both of theirs") + void nestedRollBacksRestoreTheOriginal() { + Recorder x = new Recorder(); + Recorder f = new Recorder(); + Recorder g = new Recorder(); + Tracing.install(x); + // A swaps X for F, B swaps F for G; A fails first, then B. + Tracing.Swap a = Tracing.swap(f); + Tracing.Swap b = Tracing.swap(g); + Tracing.rollBack(a); + assertTrue(Tracing.getTracer() == g, "B is still starting; its tracer stays"); + assertEquals(0, x.shutdowns, "A retired the original while B could still restore it"); + Tracing.rollBack(b); + assertTrue(Tracing.getTracer() == x, "the original was not restored"); + assertEquals(0, x.shutdowns); + assertEquals(1, f.shutdowns, "A's failed tracer"); + assertEquals(1, g.shutdowns, "B's failed tracer"); + // A failed start-up's spans are the ones that explain the failure: its + // tracer gets a window to export them, not a zero timeout. + assertTrue(g.lastShutdownMillis > 0, "a rolled-back tracer was stopped with no flush window"); + assertTrue(f.lastShutdownMillis > 0); + + // And the other order: B fails first, then A. + Tracing.Swap a2 = Tracing.swap(f = new Recorder()); + Tracing.Swap b2 = Tracing.swap(g = new Recorder()); + Tracing.rollBack(b2); + assertTrue(Tracing.getTracer() == f); + Tracing.rollBack(a2); + assertTrue(Tracing.getTracer() == x); + assertEquals(0, x.shutdowns); + assertEquals(1, f.shutdowns); + assertEquals(1, g.shutdowns); + } + + @Test + @DisplayName("a server that stops while another is starting cannot have its tracer restored") + void aStoppedServersTracerIsNotRestoredByARollBack() { + Recorder a = new Recorder(); + Recorder b = new Recorder(); + Tracing.install(a); + // B starts over A's tracer; A stops; then B fails. + Tracing.Swap claim = Tracing.swap(b); + Tracing.shutdown(a, 0); + assertEquals(1, a.shutdowns, "the stopped server's tracer kept running"); + Tracing.rollBack(claim); + assertTrue(Tracing.getTracer() == null, + "a failed start-up put back the tracer of a server that had stopped"); + assertEquals(1, a.shutdowns); + assertEquals(1, b.shutdowns); + } + + @Test + @DisplayName("a caller's tracestate alone is its own trace context") + void aStandaloneTracestateIsTheCallers() { + List lines = new ArrayList(); + lines.add("Accept: application/json"); + assertFalse(Tracing.callerTraceparent(lines)); + lines.add("tracestate: vendor=opaque"); + assertTrue(Tracing.callerTraceparent(lines), + "a traceparent of ours would have been paired with the caller's state"); + List other = new ArrayList(); + other.add("tracestatement: not-a-trace-header"); + assertFalse(Tracing.callerTraceparent(other)); + other.add("Traceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"); + assertTrue(Tracing.callerTraceparent(other)); + } + + @Test + @DisplayName("a span whose decoration throws is still ended") + void abandonedSpansAreEnded() throws Exception { + Recorder recorder = new Recorder(); + recorder.throwOnAttributes = true; + Tracing.install(recorder); + assertEquals(null, Tracing.startLambda(null, "req")); + assertEquals(1, recorder.spans.size()); + RecordedSpan span = (RecordedSpan)recorder.spans.get(0); + assertTrue(span.ended, "a span dropped after a decoration failure was never ended"); + assertTrue(span.discarded, "an incomplete span must not be exported"); + } + + private static void drain(HttpExchange ex) throws IOException { + byte[] chunk = new byte[4096]; + while(ex.getRequestBody().read(chunk) > 0) { + // Reading the request fully before answering. + } + } + + private static final class Recorder implements Tracer { + final List spans = new ArrayList(); + boolean throwOnAttributes; + + public boolean open(Config config) { + return true; + } + + public Span startSpan(String name, int kind, Span parent, String traceparent, + String tracestate) { + RecordedSpan span = new RecordedSpan(throwOnAttributes); + spans.add(span); + return span; + } + + public void flush(int timeoutMillis) { + } + + int shutdowns; + int lastShutdownMillis = -1; + + public void shutdown(int timeoutMillis) { + shutdowns++; + lastShutdownMillis = timeoutMillis; + } + + public com.codename1.backend.HttpServer.Handler relay() { + return null; + } + + public void metrics(Map out) { + } + } + + private static final class RecordedSpan extends Span { + private final boolean throwOnAttributes; + boolean ended; + boolean discarded; + String error; + final List errors = new ArrayList(); + + RecordedSpan(boolean throwOnAttributes) { + this.throwOnAttributes = throwOnAttributes; + } + + private Span attr() { + if(throwOnAttributes) { + throw new IllegalStateException("tracer bug"); + } + return this; + } + + public Span setAttribute(String key, String value) { + return attr(); + } + + public Span setAttribute(String key, long value) { + return attr(); + } + + public Span setAttribute(String key, double value) { + return attr(); + } + + public Span setAttribute(String key, boolean value) { + return attr(); + } + + public Span recordException(Throwable err) { + error = String.valueOf(err.getMessage()); + errors.add(error); + return this; + } + + public Span setError(String description) { + error = description; + return this; + } + + public Span updateName(String name) { + return this; + } + + public String getName() { + return "invoke"; + } + + public int getKind() { + return KIND_SERVER; + } + + public boolean isRecording() { + return true; + } + + public String traceparent() { + return null; + } + + public String tracestate() { + return null; + } + + public void discard() { + discarded = true; + } + + public void end() { + ended = true; + } + } +} diff --git a/maven/backend/src/test/java/com/codename1/backend/otel/OtlpTracerTest.java b/maven/backend/src/test/java/com/codename1/backend/otel/OtlpTracerTest.java new file mode 100644 index 00000000000..bbf6dd71f45 --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/otel/OtlpTracerTest.java @@ -0,0 +1,1126 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.otel; + +import com.codename1.backend.Backend; +import com.codename1.backend.Config; +import com.codename1.backend.Database; +import com.codename1.backend.HttpServer; +import com.codename1.backend.Tracing; +import com.codename1.backend.Web; + +import com.google.protobuf.ByteString; +import com.sun.net.httpserver.HttpExchange; + +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; +import io.opentelemetry.proto.trace.v1.Span; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The tracer end to end, on the JVM arm: a real server, real outbound calls, a + * real database, and a collector that decodes what arrives with the protobuf + * classes opentelemetry-proto generates -- so the hand-written encoder is judged + * by the schema itself rather than by a decoder written alongside it. + */ +class OtlpTracerTest { + private static final String TRACE = "4bf92f3577b34da6a3ce929d0e0e4736"; + private static final String CALLER_SPAN = "00f067aa0ba902b7"; + + private com.sun.net.httpserver.HttpServer collector; + private final List exports = Collections.synchronizedList(new ArrayList()); + private final List contentTypes = Collections.synchronizedList(new ArrayList()); + private final List authorizations = Collections.synchronizedList(new ArrayList()); + private final List authorizationCounts = Collections.synchronizedList(new ArrayList()); + /** When each POST reached the collector, in arrival order. */ + private final List postTimes = Collections.synchronizedList(new ArrayList()); + /** Statuses the collector answers with, in order; 200 once they run out. */ + private final java.util.concurrent.ConcurrentLinkedQueue answers = + new java.util.concurrent.ConcurrentLinkedQueue(); + /** How long the collector takes to answer; a slow one keeps the queue from emptying. */ + private volatile int collectorDelayMillis; + + @BeforeEach + void startCollector() throws IOException { + collector = com.sun.net.httpserver.HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + collector.createContext("/v1/traces", (HttpExchange exchange) -> { + exports.add(readAll(exchange.getRequestBody())); + if(collectorDelayMillis > 0) { + try { + Thread.sleep(collectorDelayMillis); + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + } + } + contentTypes.add(exchange.getRequestHeaders().getFirst("Content-Type")); + authorizations.add(String.valueOf(exchange.getRequestHeaders().getFirst("Authorization"))); + List all = exchange.getRequestHeaders().get("Authorization"); + authorizationCounts.add(Integer.valueOf(all == null ? 0 : all.size())); + postTimes.add(Long.valueOf(System.currentTimeMillis())); + Object answer = answers.poll(); + exchange.sendResponseHeaders(answer == null ? 200 : ((Integer)answer).intValue(), -1); + exchange.close(); + }); + collector.start(); + } + + @AfterEach + void stopCollector() { + Tracing.install(null); + collector.stop(0); + } + + private Properties settings(int port) { + Properties settings = new Properties(); + settings.setProperty(Config.SERVER_PORT, String.valueOf(port)); + settings.setProperty(OtlpTracer.ENDPOINT, + "http://127.0.0.1:" + collector.getAddress().getPort()); + settings.setProperty(OtlpTracer.HEADERS, "Authorization=Api-Token%20abc123"); + settings.setProperty(OtlpTracer.EXPORT_DELAY, "60000"); + return settings; + } + + @Test + @DisplayName("a request, its outbound call, the call it reaches and a statement make one trace") + void oneTraceAcrossEverything() throws Exception { + final int port = freePort(); + final Database db = Database.open(":memory:"); + db.execute("CREATE TABLE pets (id INTEGER PRIMARY KEY, name TEXT)", null); + Backend backend = Backend.builder(Config.of(settings(port), "test")) + .quiet() + .tracing(new OtlpTracer("pets")) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) throws Exception { + if(request.pathIs(ascii("/downstream"))) { + return HttpServer.Response.text(200, "down"); + } + if(!request.pathIs(ascii("/work"))) { + return null; + } + Tracing.route("/work"); + Tracing.current().setAttribute("pets.checked", 2L); + // The caller's own context: no client span may describe it. + java.util.List own = new java.util.ArrayList(); + own.add("traceparent: 00-11111111111111111111111111111111-2222222222222222-01"); + Web.request("GET", "http://127.0.0.1:" + port + "/downstream", own, null); + db.query("SELECT name FROM pets WHERE id = ?", new Object[] {Long.valueOf(7)}); + Web.Result down = Web.get("http://127.0.0.1:" + port + "/downstream"); + return HttpServer.Response.text(200, "ok " + down.getBodyAsString()); + } + }) + .start(); + try { + HttpURLConnection connection = (HttpURLConnection)new URL( + "http://127.0.0.1:" + port + "/work?secret=1").openConnection(); + connection.setRequestProperty("traceparent", "00-" + TRACE + "-" + CALLER_SPAN + "-01"); + connection.setRequestProperty("tracestate", "vendor=opaque"); + assertEquals(200, connection.getResponseCode()); + assertEquals("ok down", new String(readAll(connection.getInputStream()), "UTF-8")); + } finally { + backend.stop(); + db.close(); + } + + assertFalse(exports.isEmpty(), "stop() must flush the spans before returning"); + assertEquals("application/x-protobuf", contentTypes.get(0)); + assertEquals("Api-Token abc123", authorizations.get(0), + "OTEL_EXPORTER_OTLP_HEADERS values are percent-decoded"); + + List spans = new ArrayList(); + String service = null; + for(int iter = 0 ; iter < exports.size() ; iter++) { + ExportTraceServiceRequest request = ExportTraceServiceRequest.parseFrom( + (byte[])exports.get(iter)); + for(ResourceSpans rs : request.getResourceSpansList()) { + service = attribute(rs.getResource().getAttributesList(), "service.name"); + for(ScopeSpans ss : rs.getScopeSpansList()) { + assertEquals("com.codename1.backend", ss.getScope().getName()); + spans.addAll(ss.getSpansList()); + } + } + } + assertEquals("pets", service); + + Span work = find(spans, "GET /work", Span.SpanKind.SPAN_KIND_SERVER); + Span query = find(spans, "SELECT", Span.SpanKind.SPAN_KIND_CLIENT); + // By KIND as well as name: the downstream request is "GET" too, since no + // route names it, and it is a server span. + Span outbound = find(spans, "GET", Span.SpanKind.SPAN_KIND_CLIENT); + Span downstream = null; + int clientGets = 0; + for(Object o : spans) { + Span s = (Span)o; + if("GET".equals(s.getName()) && s.getKind() == Span.SpanKind.SPAN_KIND_CLIENT) { + clientGets++; + } + if("GET".equals(s.getName()) && s.getKind() == Span.SpanKind.SPAN_KIND_SERVER + && TRACE.equals(hex(s.getTraceId()))) { + downstream = s; + } + } + assertEquals(1, clientGets, + "a client span was recorded for the call that carried the caller's own traceparent"); + assertTrue(downstream != null, "the downstream request's span in this trace"); + + // The server span continues the caller's trace, under the caller's span. + assertEquals(TRACE, hex(work.getTraceId())); + assertEquals(CALLER_SPAN, hex(work.getParentSpanId())); + assertEquals(Span.SpanKind.SPAN_KIND_SERVER, work.getKind()); + assertEquals("vendor=opaque", work.getTraceState()); + assertEquals(0x301, work.getFlags(), "sampled, with a parent known to be remote"); + assertEquals("/work", attribute(work.getAttributesList(), "http.route")); + assertEquals("/work", attribute(work.getAttributesList(), "url.path"), + "the query string is never recorded"); + assertEquals("200", attribute(work.getAttributesList(), "http.response.status_code")); + assertEquals("2", attribute(work.getAttributesList(), "pets.checked")); + assertTrue(work.getEndTimeUnixNano() >= work.getStartTimeUnixNano()); + + // The statement and the outbound call are its children. + assertEquals(TRACE, hex(query.getTraceId())); + assertEquals(hex(work.getSpanId()), hex(query.getParentSpanId())); + assertEquals(Span.SpanKind.SPAN_KIND_CLIENT, query.getKind()); + assertEquals("sqlite", attribute(query.getAttributesList(), "db.system")); + assertEquals("SELECT name FROM pets WHERE id = ?", + attribute(query.getAttributesList(), "db.query.text"), + "the statement is recorded, the bound value is not"); + assertEquals(hex(work.getSpanId()), hex(outbound.getParentSpanId())); + assertEquals(Span.SpanKind.SPAN_KIND_CLIENT, outbound.getKind()); + // Children sit inside their parent on ONE timeline. Each span reading the + // wall clock for itself put them up to a millisecond apart, and a child + // was reported ending after the request that contains it. + for(Span child : new Span[] {query, outbound}) { + assertTrue(child.getStartTimeUnixNano() >= work.getStartTimeUnixNano(), + child.getName() + " starts before its parent"); + assertTrue(child.getEndTimeUnixNano() <= work.getEndTimeUnixNano(), + child.getName() + " ends after its parent"); + } + + // And the traceparent the outbound call carried made the request it + // reached a child of THAT span -- propagation, observed from the far side. + assertEquals(TRACE, hex(downstream.getTraceId())); + assertEquals(hex(outbound.getSpanId()), hex(downstream.getParentSpanId())); + assertEquals("vendor=opaque", downstream.getTraceState(), "tracestate travels too"); + } + + @Test + @DisplayName("a failing handler is an error span with the exception recorded") + void errorsAreRecorded() throws Exception { + int port = freePort(); + Backend backend = Backend.builder(Config.of(settings(port), "test")) + .quiet() + .tracing(new OtlpTracer()) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) throws Exception { + StringBuilder huge = new StringBuilder("boom"); + while(huge.length() < 20000) { + huge.append(" and more"); + } + throw new IllegalStateException(huge.toString()); + } + }) + .start(); + try { + HttpURLConnection connection = (HttpURLConnection)new URL( + "http://127.0.0.1:" + port + "/x").openConnection(); + assertEquals(500, connection.getResponseCode()); + } finally { + backend.stop(); + } + ExportTraceServiceRequest request = ExportTraceServiceRequest.parseFrom( + (byte[])exports.get(0)); + Span span = request.getResourceSpans(0).getScopeSpans(0).getSpans(0); + assertEquals("unknown_service", + attribute(request.getResourceSpans(0).getResource().getAttributesList(), + "service.name")); + assertEquals(io.opentelemetry.proto.trace.v1.Status.StatusCode.STATUS_CODE_ERROR, + span.getStatus().getCode()); + assertEquals("exception", span.getEvents(0).getName()); + assertEquals(OtelSpan.MAX_VALUE_LENGTH, span.getStatus().getMessage().length(), + "the status description is bounded like the event attribute"); + assertEquals("java.lang.IllegalStateException", + attribute(span.getEvents(0).getAttributesList(), "exception.type")); + assertEquals(0, span.getParentSpanId().size(), "a root span has no parent"); + assertEquals(1, span.getFlags(), + "a root has no parent context, so its remoteness is not claimed either way"); + assertEquals(32, hex(span.getTraceId()).length()); + } + + @Test + @DisplayName("an excluded exception attribute is recorded neither in the event nor the status") + void excludedExceptionAttributes() throws Exception { + int port = freePort(); + Properties settings = settings(port); + settings.setProperty(OtlpTracer.ATTRIBUTES_EXCLUDE, "exception.message"); + Backend backend = Backend.builder(Config.of(settings, "test")) + .quiet() + .tracing(new OtlpTracer()) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) throws Exception { + throw new IllegalStateException("card 4111 declined"); + } + }) + .start(); + try { + HttpURLConnection connection = (HttpURLConnection)new URL( + "http://127.0.0.1:" + port + "/x").openConnection(); + assertEquals(500, connection.getResponseCode()); + } finally { + backend.stop(); + } + Span span = ExportTraceServiceRequest.parseFrom((byte[])exports.get(0)) + .getResourceSpans(0).getScopeSpans(0).getSpans(0); + assertEquals("exception", span.getEvents(0).getName()); + assertNull(attribute(span.getEvents(0).getAttributesList(), "exception.message"), + "an excluded attribute was exported on the exception event"); + assertEquals("java.lang.IllegalStateException", + attribute(span.getEvents(0).getAttributesList(), "exception.type")); + assertEquals("java.lang.IllegalStateException", span.getStatus().getMessage(), + "the status description carried the excluded message"); + } + + @Test + @DisplayName("installing a tracer shuts down the one it replaces, and only that one") + void replacingATracerShutsDownThePreviousOne() { + final int[] shutdowns = new int[2]; + ThrowingTracer first = new ThrowingTracer() { + public void shutdown(int timeoutMillis) { + shutdowns[0]++; + } + }; + ThrowingTracer second = new ThrowingTracer() { + public void shutdown(int timeoutMillis) { + shutdowns[1]++; + } + }; + Tracing.install(first); + Tracing.install(first); + assertEquals(0, shutdowns[0], "re-installing the same tracer stopped it"); + Tracing.install(second); + assertEquals(1, shutdowns[0], "the replaced tracer was left running"); + Tracing.install(null); + assertEquals(1, shutdowns[1], "turning tracing off left the tracer running"); + assertEquals(1, shutdowns[0]); + } + + @Test + @DisplayName("flush returns once what was queued before it has gone, though the queue never empties") + void flushWaitsOnlyForWhatItFound() throws Exception { + collectorDelayMillis = 20; + Properties settings = settings(freePort()); + settings.setProperty(OtlpTracer.BATCH_SIZE, "1"); + settings.setProperty(OtlpTracer.QUEUE_SIZE, "5"); + final OtlpTracer tracer = new OtlpTracer(); + assertTrue(tracer.open(Config.of(settings, "test"))); + final java.util.concurrent.atomic.AtomicBoolean stop = + new java.util.concurrent.atomic.AtomicBoolean(); + Thread busy = new Thread(() -> { + // Spans ending far faster than one export per 20ms: the queue is never + // empty again for as long as this runs. + while(!stop.get()) { + tracer.startSpan("busy", com.codename1.backend.Span.KIND_INTERNAL, + null, null, null).end(); + Thread.yield(); + } + }); + busy.start(); + try { + Thread.sleep(100); + long started = System.currentTimeMillis(); + tracer.flush(10000); + long took = System.currentTimeMillis() - started; + assertTrue(took < 5000, "flush waited " + took + + "ms for spans queued after it was called"); + } finally { + stop.set(true); + busy.join(); + tracer.shutdown(0); + } + } + + @Test + @DisplayName("a header value that cannot be sent is refused when the tracer opens") + void anUnsendableHeaderIsRefusedAtStartup() throws Exception { + Properties settings = settings(freePort()); + settings.setProperty(OtlpTracer.HEADERS, "Authorization=token%0Aextra"); + IOException refused = assertThrows(IOException.class, + () -> new OtlpTracer().open(Config.of(settings, "test"))); + assertTrue(refused.getMessage().contains("Authorization"), refused.getMessage()); + assertFalse(refused.getMessage().contains("token"), + "the refusal quoted the credential: " + refused.getMessage()); + settings.setProperty(OtlpTracer.HEADERS, "Host=collector.example"); + assertThrows(IOException.class, () -> new OtlpTracer().open(Config.of(settings, "test"))); + settings.setProperty(OtlpTracer.HEADERS, "Content-Type=application/json"); + assertThrows(IOException.class, () -> new OtlpTracer().open(Config.of(settings, "test")), + "a configured Content-Type went out as a second one"); + } + + @Test + @DisplayName("the relay refuses a negative timestamp instead of forwarding it") + void aNegativeTimestampIsRefused() throws Exception { + java.util.Map request = (java.util.Map)com.codename1.backend.Json.parse( + "{\"resourceSpans\":[{\"scopeSpans\":[{\"spans\":[{\"traceId\":\"" + + TRACE + "\",\"spanId\":\"" + CALLER_SPAN + "\",\"name\":\"x\"," + + "\"startTimeUnixNano\":\"-1\"}]}]}]}"); + IOException refused = assertThrows(IOException.class, + () -> OtlpSchema.sanitize(request)); + assertTrue(refused.getMessage().contains("startTimeUnixNano"), refused.getMessage()); + } + + @Test + @DisplayName("stopping a server leaves a tracer it did not install running") + void stoppingAServerLeavesAnotherTracerAlone() throws Exception { + int port = freePort(); + Backend backend = Backend.builder(Config.of(settings(port), "test")) + .quiet() + .tracing(new OtlpTracer()) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) { + return null; + } + }) + .start(); + final int[] shutdowns = new int[1]; + ThrowingTracer replacement = new ThrowingTracer() { + public void shutdown(int timeoutMillis) { + shutdowns[0]++; + } + }; + try { + Tracing.install(replacement); + } finally { + backend.stop(); + } + assertSame(replacement, Tracing.getTracer(), + "stopping the server uninstalled a tracer it never installed"); + assertEquals(0, shutdowns[0], "stopping the server shut down another tracer"); + } + + @Test + @DisplayName("a span from a replaced tracer is never adopted as a parent") + void aReplacedTracersSpanIsNotAParent() throws Exception { + OtlpTracer before = new OtlpTracer(); + OtlpTracer after = new OtlpTracer(); + assertTrue(before.open(Config.of(settings(freePort()), "test"))); + assertTrue(after.open(Config.of(settings(freePort()), "test"))); + try { + OtelSpan old = (OtelSpan)before.startSpan("request", com.codename1.backend.Span.KIND_SERVER, + null, null, null); + OtelSpan child = (OtelSpan)after.startSpan("query", com.codename1.backend.Span.KIND_CLIENT, + old, null, null); + assertFalse(old.traceHi == child.traceHi && old.traceLo == child.traceLo, + "the new tracer's span joined the replaced tracer's trace"); + assertEquals(0, child.parentId); + OtelSpan grandchild = (OtelSpan)after.startSpan("row", com.codename1.backend.Span.KIND_INTERNAL, + child, null, null); + assertEquals(child.spanId, grandchild.parentId, "its own spans still nest"); + } finally { + before.shutdown(0); + after.shutdown(0); + } + } + + @Test + @DisplayName("a relay path is matched as the server normalizes request paths") + void relayPathIsNormalized() throws Exception { + int port = freePort(); + Properties settings = settings(port); + settings.setProperty(OtlpTracer.RELAY, "true"); + settings.setProperty(OtlpTracer.RELAY_PATH, "/otel/%74races"); + Backend backend = Backend.builder(Config.of(settings, "test")) + .quiet() + .tracing(new OtlpTracer("pets")) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) { + return HttpServer.Response.text(404, "not the relay"); + } + }) + .start(); + try { + assertEquals(200, post(port, "/otel/traces", "{\"resourceSpans\":[]}", null), + "the relay configured as /otel/%74races never answered /otel/traces"); + assertEquals(200, post(port, "/otel/%74races", "{\"resourceSpans\":[]}", null), + "nor the spelling that was configured"); + } finally { + backend.stop(); + } + } + + @Test + @DisplayName("a blank service name falls back to unknown_service") + void aBlankServiceNameIsUnknownService() throws Exception { + Properties settings = settings(freePort()); + settings.setProperty(OtlpTracer.SERVICE_NAME, " "); + OtlpTracer tracer = new OtlpTracer(" "); + assertTrue(tracer.open(Config.of(settings, "test"))); + try { + tracer.startSpan("x", com.codename1.backend.Span.KIND_INTERNAL, null, null, null).end(); + tracer.flush(5000); + } finally { + tracer.shutdown(0); + } + ExportTraceServiceRequest request = ExportTraceServiceRequest.parseFrom( + (byte[])exports.get(0)); + assertEquals("unknown_service", + attribute(request.getResourceSpans(0).getResource().getAttributesList(), + "service.name")); + } + + @Test + @DisplayName("a server that fails to start leaves the tracer it found installed and running") + void aFailedStartKeepsTheInstalledTracer() throws Exception { + final int[] shutdowns = new int[1]; + ThrowingTracer existing = new ThrowingTracer() { + public void shutdown(int timeoutMillis) { + shutdowns[0]++; + } + }; + Tracing.install(existing); + // The port is taken, so this start-up fails after its tracer went in. + java.net.ServerSocket blocker = new java.net.ServerSocket(0); + try { + int port = blocker.getLocalPort(); + assertThrows(Exception.class, () -> Backend.builder(Config.of(settings(port), "test")) + .quiet() + .tracing(new OtlpTracer()) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) { + return null; + } + }) + .start()); + } finally { + blocker.close(); + } + assertSame(existing, Tracing.getTracer(), + "a failed start-up left the process with no tracer at all"); + assertEquals(0, shutdowns[0], "a failed start-up shut down a tracer that was working"); + } + + @Test + @DisplayName("shutdown stops within its window instead of draining a slow collector's queue") + void shutdownIsBoundedBySlowCollectors() throws Exception { + collectorDelayMillis = 1500; + Properties settings = settings(freePort()); + settings.setProperty(OtlpTracer.BATCH_SIZE, "1"); + OtlpTracer tracer = new OtlpTracer(); + assertTrue(tracer.open(Config.of(settings, "test"))); + for(int i = 0 ; i < 10 ; i++) { + tracer.startSpan("s" + i, com.codename1.backend.Span.KIND_INTERNAL, null, null, null).end(); + } + long started = System.currentTimeMillis(); + tracer.shutdown(200); + assertTrue(System.currentTimeMillis() - started < 3000, "shutdown overran its window"); + // Draining would post all ten, one every 1.5s; bounded, the post already in + // flight is the last. + Thread.sleep(5000); + assertTrue(exports.size() <= 2, + "the stopped exporter kept posting to the collector: " + exports.size() + " posts"); + } + + @Test + @DisplayName("a retryable failure is retried once, after a backoff, without sleeping") + void aRetryableFailureIsRetriedOnceFromTheQueue() throws Exception { + Properties settings = settings(freePort()); + settings.setProperty(OtlpTracer.EXPORT_DELAY, "200"); + answers.add(Integer.valueOf(503)); + OtlpTracer tracer = new OtlpTracer(); + assertTrue(tracer.open(Config.of(settings, "test"))); + try { + tracer.startSpan("once", com.codename1.backend.Span.KIND_INTERNAL, null, null, null).end(); + tracer.flush(5000); + java.util.Map metrics = new java.util.LinkedHashMap(); + tracer.metrics(metrics); + assertEquals(2, exports.size(), "the 503 and the retry that succeeded"); + assertEquals(Long.valueOf(1), metrics.get("spansExported")); + + // Refused again on its retry: dropped, never tried a third time. + answers.add(Integer.valueOf(503)); + answers.add(Integer.valueOf(503)); + tracer.startSpan("twice", com.codename1.backend.Span.KIND_INTERNAL, null, null, null).end(); + // flush returns once the span is exported OR dropped: here, dropped. + tracer.flush(5000); + metrics.clear(); + tracer.metrics(metrics); + assertEquals(4, exports.size(), "a span was retried more than once"); + assertEquals(Long.valueOf(1), metrics.get("spansDropped")); + } finally { + tracer.shutdown(0); + } + } + + @Test + @DisplayName("the backoff holds even after a batch that used its retry is dropped") + void backoffHoldsAfterADroppedBatch() throws Exception { + Properties settings = settings(freePort()); + settings.setProperty(OtlpTracer.EXPORT_DELAY, "500"); + answers.add(Integer.valueOf(429)); + answers.add(Integer.valueOf(429)); + OtlpTracer tracer = new OtlpTracer(); + assertTrue(tracer.open(Config.of(settings, "test"))); + try { + tracer.startSpan("first", com.codename1.backend.Span.KIND_INTERNAL, null, null, null).end(); + tracer.flush(5000); // posted, retried, dropped + tracer.startSpan("second", com.codename1.backend.Span.KIND_INTERNAL, null, null, null).end(); + tracer.flush(5000); + assertEquals(3, postTimes.size(), String.valueOf(postTimes)); + long afterDrop = ((Long)postTimes.get(2)).longValue() - ((Long)postTimes.get(1)).longValue(); + assertTrue(afterDrop >= 400, "the next batch went out " + afterDrop + + "ms after a 429, not after the backoff"); + } finally { + tracer.shutdown(0); + } + } + + @Test + @DisplayName("relay payloads that were never posted keep their own retry") + void unsentRelayPayloadsKeepTheirRetry() throws Exception { + // One round of two: A's first POST fails, and A and the unsent B go back. + // B's own first POST then fails too, and must still get its retry. + answers.add(Integer.valueOf(503)); // A + answers.add(Integer.valueOf(200)); // A, retried + answers.add(Integer.valueOf(503)); // B, first real attempt + answers.add(Integer.valueOf(200)); // B, retried + BatchExporter exporter = new BatchExporter( + "http://127.0.0.1:" + collector.getAddress().getPort() + "/v1/traces", + new ArrayList(), false, new java.util.LinkedHashMap(), 16, 4, 100, 1 << 20); + assertTrue(exporter.addRelayed("{\"a\":1}".getBytes("UTF-8"), "application/json")); + assertTrue(exporter.addRelayed("{\"b\":1}".getBytes("UTF-8"), "application/json")); + exporter.start(); + try { + exporter.flush(10000); + java.util.Map metrics = new java.util.LinkedHashMap(); + exporter.metrics(metrics); + assertEquals(Long.valueOf(2), metrics.get("clientExportsRelayed"), + "a payload that was never sent was dropped on its first failure"); + assertEquals(Long.valueOf(0), metrics.get("clientExportsDropped")); + } finally { + exporter.shutdown(0); + } + } + + @Test + @DisplayName("a request that stops the server still exports its own span") + void theStoppingRequestKeepsItsSpan() throws Exception { + int port = freePort(); + final java.util.concurrent.atomic.AtomicReference server = + new java.util.concurrent.atomic.AtomicReference(); + Backend backend = Backend.builder(Config.of(settings(port), "test")) + .quiet() + .tracing(new OtlpTracer()) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) { + if(request.getTarget().startsWith("/shutdown")) { + ((Backend)server.get()).stop(); + return HttpServer.Response.text(200, "stopping"); + } + return null; + } + }) + .start(); + server.set(backend); + HttpURLConnection connection = (HttpURLConnection)new URL( + "http://127.0.0.1:" + port + "/shutdown").openConnection(); + assertEquals(200, connection.getResponseCode()); + long deadline = System.currentTimeMillis() + 10000; + boolean found = false; + while(!found && System.currentTimeMillis() < deadline) { + for(int iter = 0 ; iter < exports.size() && !found ; iter++) { + ExportTraceServiceRequest sent = ExportTraceServiceRequest.parseFrom((byte[])exports.get(iter)); + for(ResourceSpans rs : sent.getResourceSpansList()) { + for(io.opentelemetry.proto.trace.v1.ScopeSpans ss : rs.getScopeSpansList()) { + for(Span span : ss.getSpansList()) { + found |= "/shutdown".equals(attribute(span.getAttributesList(), "url.path")); + } + } + } + } + if(!found) { + Thread.sleep(50); + } + } + assertTrue(found, "the request that stopped the server lost its span"); + } + + @Test + @DisplayName("the relay refuses too many spans before validating them") + void relaySpanCapComesFirst() throws Exception { + int port = freePort(); + Properties settings = settings(port); + settings.setProperty(OtlpTracer.RELAY, "true"); + settings.setProperty(OtlpTracer.RELAY_MAX_SPANS, "1"); + Backend backend = Backend.builder(Config.of(settings, "test")) + .quiet() + .tracing(new OtlpTracer("pets")) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) { + return null; + } + }) + .start(); + try { + // Two spans, both with ids the sanitizer would refuse (400). Counting + // first answers 413 without building the sanitized copy at all. + String body = "{\"resourceSpans\":[{\"scopeSpans\":[{\"spans\":[" + + "{\"traceId\":\"nothex\",\"spanId\":\"x\"}," + + "{\"traceId\":\"nothex\",\"spanId\":\"y\"}]}]}]}"; + assertEquals(413, post(port, "/otel/v1/traces", body, null)); + } finally { + backend.stop(); + } + } + + @Test + @DisplayName("OTEL_SDK_DISABLED leaves the server untraced and the collector untouched") + void disabledAtRunTime() throws Exception { + int port = freePort(); + Properties settings = settings(port); + settings.setProperty(OtlpTracer.DISABLED, "true"); + Backend backend = Backend.builder(Config.of(settings, "test")) + .quiet() + .tracing(new OtlpTracer("pets")) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) throws Exception { + assertNull(Tracing.currentTraceparent()); + return HttpServer.Response.text(200, "ok"); + } + }) + .start(); + try { + assertFalse(Tracing.isEnabled()); + HttpURLConnection connection = (HttpURLConnection)new URL( + "http://127.0.0.1:" + port + "/x").openConnection(); + assertEquals(200, connection.getResponseCode()); + } finally { + backend.stop(); + } + assertTrue(exports.isEmpty()); + } + + @Test + @DisplayName("the traces-specific headers replace the generic ones rather than adding to them") + void signalHeadersReplaceGenericOnes() throws Exception { + int port = freePort(); + Properties settings = settings(port); + settings.setProperty(OtlpTracer.TRACES_HEADERS, "Authorization=Api-Token%20traces"); + Backend backend = Backend.builder(Config.of(settings, "test")) + .quiet() + .tracing(new OtlpTracer("pets")) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) throws Exception { + return HttpServer.Response.text(200, "ok"); + } + }) + .start(); + try { + HttpURLConnection connection = (HttpURLConnection)new URL( + "http://127.0.0.1:" + port + "/x").openConnection(); + assertEquals(200, connection.getResponseCode()); + } finally { + backend.stop(); + } + assertEquals("Api-Token traces", authorizations.get(0)); + assertEquals(Integer.valueOf(1), authorizationCounts.get(0), + "two Authorization headers went to the collector"); + } + + @Test + @DisplayName("a tracer that throws while decorating a query does not fail the query") + void aBrokenTracerCannotFailAQuery() throws Exception { + Database db = Database.open(":memory:"); + db.execute("CREATE TABLE t (v INTEGER)", null); + db.execute("INSERT INTO t (v) VALUES (1)", null); + Tracing.install(new ThrowingTracer()); + try { + assertEquals(1, db.query("SELECT v FROM t", null).size(), + "the rows a query fetched must reach the caller whatever the tracer does"); + // And inside a span, where starting the statement's span first asks the + // parent's kind -- which this tracer's spans throw from. + Object nested = Tracing.inSpan("work", span -> db.query("SELECT v FROM t", null).size()); + assertEquals(Integer.valueOf(1), nested); + } finally { + Tracing.install(null); + db.close(); + } + } + + @Test + @DisplayName("a non-ASCII relay token is not satisfied by its ASCII lookalike") + void relayTokenComparesEveryCharacter() throws Exception { + int port = freePort(); + Properties settings = settings(port); + settings.setProperty(OtlpTracer.RELAY, "true"); + settings.setProperty(OtlpTracer.RELAY_TOKEN, "s\u00ebcret"); + Backend backend = Backend.builder(Config.of(settings, "test")) + .quiet() + .tracing(new OtlpTracer("pets")) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) throws Exception { + return null; + } + }) + .start(); + try { + // The old comparison folded every non-ASCII character to '?'. + assertEquals(401, post(port, "/otel/v1/traces", "{\"resourceSpans\":[]}", "s?cret")); + } finally { + backend.stop(); + } + } + + @Test + @DisplayName("a tracer that throws while a route is named does not fail the request") + void aBrokenTracerCannotFailARoute() throws Exception { + Tracing.install(new ThrowingTracer()); + try { + Object result = Tracing.inSpan("work", span -> { + // What every generated router calls on a matched request. + Tracing.route("/pets/{id}"); + return "handled"; + }); + assertEquals("handled", result); + } finally { + Tracing.install(null); + } + } + + /** A tracer whose spans throw from every decoration. */ + private static class ThrowingTracer implements com.codename1.backend.Tracer { + public boolean open(Config config) { + return true; + } + + public com.codename1.backend.Span startSpan(String name, int kind, + com.codename1.backend.Span parent, String traceparent, String tracestate) { + return new com.codename1.backend.Span() { + public com.codename1.backend.Span setAttribute(String key, String value) { + throw new IllegalStateException("tracer bug"); + } + + public com.codename1.backend.Span setAttribute(String key, long value) { + throw new IllegalStateException("tracer bug"); + } + + public com.codename1.backend.Span setAttribute(String key, double value) { + throw new IllegalStateException("tracer bug"); + } + + public com.codename1.backend.Span setAttribute(String key, boolean value) { + throw new IllegalStateException("tracer bug"); + } + + public com.codename1.backend.Span recordException(Throwable error) { + throw new IllegalStateException("tracer bug"); + } + + public com.codename1.backend.Span setError(String description) { + throw new IllegalStateException("tracer bug"); + } + + public com.codename1.backend.Span updateName(String name) { + return this; + } + + public String getName() { + return "x"; + } + + public int getKind() { + throw new IllegalStateException("tracer bug"); + } + + public boolean isRecording() { + return true; + } + + public String traceparent() { + return null; + } + + public String tracestate() { + return null; + } + + public void discard() { + } + + public void end() { + } + }; + } + + public void flush(int timeoutMillis) { + } + + public void shutdown(int timeoutMillis) { + } + + public HttpServer.Handler relay() { + return null; + } + + public void metrics(java.util.Map out) { + } + } + + @Test + @DisplayName("spans a collector rejects in a partial success are not counted as exported") + void partialSuccessIsCounted() throws Exception { + collector.removeContext("/v1/traces"); + collector.createContext("/v1/traces", (HttpExchange exchange) -> { + readAll(exchange.getRequestBody()); + byte[] answer = io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse + .newBuilder() + .setPartialSuccess(io.opentelemetry.proto.collector.trace.v1 + .ExportTracePartialSuccess.newBuilder() + .setRejectedSpans(1).setErrorMessage("span too old").build()) + .build().toByteArray(); + exchange.getResponseHeaders().add("Content-Type", "application/x-protobuf"); + exchange.sendResponseHeaders(200, answer.length); + exchange.getResponseBody().write(answer); + exchange.close(); + }); + int port = freePort(); + Backend backend = Backend.builder(Config.of(settings(port), "test")) + .quiet() + .tracing(new OtlpTracer("pets")) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) throws Exception { + return HttpServer.Response.text(200, "ok"); + } + }) + .start(); + java.util.Map metrics; + try { + HttpURLConnection connection = (HttpURLConnection)new URL( + "http://127.0.0.1:" + port + "/x").openConnection(); + assertEquals(200, connection.getResponseCode()); + Tracing.getTracer().flush(5000); + metrics = backend.getServer().getMetrics(); + } finally { + backend.stop(); + } + assertEquals(Long.valueOf(1), metrics.get("spansRejected"), String.valueOf(metrics)); + assertEquals(Long.valueOf(0), metrics.get("spansExported"), String.valueOf(metrics)); + } + + @Test + @DisplayName("http/json exports the same tree as OTLP/JSON") + void jsonProtocol() throws Exception { + int port = freePort(); + Properties settings = settings(port); + settings.setProperty(OtlpTracer.PROTOCOL, "http/json"); + Backend backend = Backend.builder(Config.of(settings, "test")) + .quiet() + .tracing(new OtlpTracer("pets")) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) throws Exception { + return HttpServer.Response.text(200, "ok"); + } + }) + .start(); + try { + HttpURLConnection connection = (HttpURLConnection)new URL( + "http://127.0.0.1:" + port + "/x").openConnection(); + assertEquals(200, connection.getResponseCode()); + } finally { + backend.stop(); + } + assertEquals("application/json", contentTypes.get(0)); + String json = new String((byte[])exports.get(0), "UTF-8"); + assertTrue(json.startsWith("{\"resourceSpans\":[{\"resource\":{\"attributes\":"), json); + assertTrue(json.contains("\"kind\":2"), json); + // And it is the SAME export: re-encoding the JSON through the schema gives + // bytes the protobuf classes read back as the span that was sent. + byte[] proto = OtlpSchema.protobuf(com.codename1.backend.Json.parseObject(json)); + Span span = ExportTraceServiceRequest.parseFrom(proto) + .getResourceSpans(0).getScopeSpans(0).getSpans(0); + assertEquals("GET", span.getName()); + } + + @Test + @DisplayName("the relay re-encodes a client's JSON export and refuses what is not one") + void relay() throws Exception { + int port = freePort(); + Properties settings = settings(port); + settings.setProperty(OtlpTracer.RELAY, "true"); + settings.setProperty(OtlpTracer.RELAY_TOKEN, "s3cret"); + Backend backend = Backend.builder(Config.of(settings, "test")) + .quiet() + .tracing(new OtlpTracer("pets")) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) throws Exception { + return null; + } + }) + .start(); + String client = "{\"resourceSpans\":[{\"resource\":{\"attributes\":[{\"key\":" + + "\"service.name\",\"value\":{\"stringValue\":\"app\"}}]},\"scopeSpans\":[{" + + "\"scope\":{\"name\":\"com.codename1.io\"},\"spans\":[{\"traceId\":\"" + + TRACE + "\",\"spanId\":\"" + CALLER_SPAN + "\",\"name\":\"GET /pets\"," + + "\"kind\":3,\"startTimeUnixNano\":\"1700000000000000000\"," + + "\"endTimeUnixNano\":\"1700000000500000000\",\"smuggled\":\"x\"," + + "\"attributes\":[{\"key\":\"http.response.status_code\",\"value\":" + + "{\"intValue\":\"200\"}}]}]}]}]}"; + try { + assertEquals(401, post(port, "/otel/v1/traces", client, null)); + assertEquals(401, post(port, "/otel/v1/traces", client, "s3cre?"), + "a wrong token is refused"); + assertEquals(415, postTyped(port, "/otel/v1/traces", client, "s3cret", "text/plain")); + assertEquals(400, post(port, "/otel/v1/traces", + client.replace(TRACE, "nothex"), "s3cret")); + assertEquals(400, post(port, "/otel/v1/traces", + client.replace(TRACE, "00000000000000000000000000000000"), "s3cret"), + "an all-zero trace id is refused before the relay answers 200"); + assertEquals(400, post(port, "/otel/v1/traces", + client.replace("\"traceId\":\"" + TRACE + "\",", ""), "s3cret"), + "a span without a trace id is refused"); + assertEquals(400, post(port, "/otel/v1/traces", + client.replace("{\"intValue\":\"200\"}", + "{\"intValue\":\"200\",\"stringValue\":\"x\"}"), "s3cret"), + "an AnyValue holding two alternatives of its oneof is refused"); + assertEquals(200, post(port, "/otel/v1/traces", client, "s3cret")); + } finally { + backend.stop(); + } + // The relay's own request is not traced, so the one export is the client's. + assertEquals(1, exports.size()); + ExportTraceServiceRequest request = ExportTraceServiceRequest.parseFrom( + (byte[])exports.get(0)); + ResourceSpans rs = request.getResourceSpans(0); + assertEquals("app", attribute(rs.getResource().getAttributesList(), "service.name"), + "the client's resource is forwarded as the client's"); + Span span = rs.getScopeSpans(0).getSpans(0); + assertEquals(TRACE, hex(span.getTraceId())); + assertEquals("GET /pets", span.getName()); + assertEquals(1700000000500000000L, span.getEndTimeUnixNano()); + assertEquals("200", attribute(span.getAttributesList(), "http.response.status_code")); + assertEquals("Api-Token abc123", authorizations.get(0), + "the server adds the collector credential the app never had"); + } + + // ------------------------------------------------------------------ + + private static Span find(List spans, String name, Span.SpanKind kind) { + for(int iter = 0 ; iter < spans.size() ; iter++) { + Span span = (Span)spans.get(iter); + if(name.equals(span.getName()) && span.getKind() == kind) { + return span; + } + } + throw new AssertionError("no " + kind + " span named " + name + " in " + spans); + } + + private static String attribute(List attributes, String key) { + for(int iter = 0 ; iter < attributes.size() ; iter++) { + KeyValue kv = (KeyValue)attributes.get(iter); + if(kv.getKey().equals(key)) { + if(kv.getValue().hasIntValue()) { + return String.valueOf(kv.getValue().getIntValue()); + } + return kv.getValue().getStringValue(); + } + } + return null; + } + + private static String hex(ByteString bytes) { + StringBuilder out = new StringBuilder(); + for(int iter = 0 ; iter < bytes.size() ; iter++) { + out.append(String.format("%02x", bytes.byteAt(iter) & 0xff)); + } + return out.toString(); + } + + private static int post(int port, String path, String body, String token) throws IOException { + return postTyped(port, path, body, token, "application/json"); + } + + private static int postTyped(int port, String path, String body, String token, String type) + throws IOException { + HttpURLConnection connection = (HttpURLConnection)new URL( + "http://127.0.0.1:" + port + path).openConnection(); + connection.setRequestMethod("POST"); + connection.setDoOutput(true); + connection.setRequestProperty("Content-Type", type); + if(token != null) { + connection.setRequestProperty("X-CN1-Telemetry-Token", token); + } + OutputStream out = connection.getOutputStream(); + out.write(body.getBytes("UTF-8")); + out.close(); + return connection.getResponseCode(); + } + + private static byte[] ascii(String value) { + try { + return value.getBytes("US-ASCII"); + } catch (IOException err) { + throw new IllegalStateException(err); + } + } + + private static byte[] readAll(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] chunk = new byte[4096]; + int n; + while((n = in.read(chunk)) > 0) { + out.write(chunk, 0, n); + } + in.close(); + return out.toByteArray(); + } + + private static int freePort() throws IOException { + java.net.ServerSocket probe = new java.net.ServerSocket(0); + try { + return probe.getLocalPort(); + } finally { + probe.close(); + } + } +} diff --git a/maven/backend/src/test/java/com/codename1/backend/otel/TraceContextTest.java b/maven/backend/src/test/java/com/codename1/backend/otel/TraceContextTest.java new file mode 100644 index 00000000000..da47e24c278 --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/otel/TraceContextTest.java @@ -0,0 +1,584 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.otel; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** W3C Trace Context parsing, which is input from the internet. */ +class TraceContextTest { + private static final String VALID = + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; + + @Test + @DisplayName("the specification's own example parses, and formats back to itself") + void roundTrip() { + TraceContext parsed = TraceContext.parse(VALID); + assertNotNull(parsed); + assertTrue(parsed.sampled()); + assertEquals(VALID, TraceContext.format(parsed.traceHi, parsed.traceLo, + parsed.spanId, parsed.sampled())); + } + + @Test + @DisplayName("an unsampled parent is read as unsampled") + void unsampled() { + TraceContext parsed = TraceContext.parse( + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00"); + assertNotNull(parsed); + assertFalse(parsed.sampled()); + } + + @Test + @DisplayName("everything the specification says to ignore is ignored") + void malformedIsRefused() { + // Upper case hex: the specification requires lower case. + assertNull(TraceContext.parse("00-4BF92F3577B34DA6A3CE929D0E0E4736-00f067aa0ba902b7-01")); + // The forbidden version. + assertNull(TraceContext.parse("ff-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01")); + // All-zero ids. + assertNull(TraceContext.parse("00-00000000000000000000000000000000-00f067aa0ba902b7-01")); + assertNull(TraceContext.parse("00-4bf92f3577b34da6a3ce929d0e0e4736-0000000000000000-01")); + // Version 00 has exactly four fields. + assertNull(TraceContext.parse(VALID + "-extra")); + // Truncated, and wrong separators. + assertNull(TraceContext.parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7")); + assertNull(TraceContext.parse("00_4bf92f3577b34da6a3ce929d0e0e4736_00f067aa0ba902b7_01")); + assertNull(TraceContext.parse("00-4bf92f3577b34da6a3ce929d0e0e473g-00f067aa0ba902b7-01")); + assertNull(TraceContext.parse(null)); + assertNull(TraceContext.parse("")); + } + + @Test + @DisplayName("a later version is read for the fields this one knows") + void futureVersion() { + TraceContext parsed = TraceContext.parse( + "01-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01-whatever"); + assertNotNull(parsed); + // ...but only when the flags end where they should. + assertNull(TraceContext.parse( + "01-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01x")); + } + + @Test + @DisplayName("tracestate passes through when sound and is dropped when not") + void tracestate() { + assertEquals("rojo=00f067aa0ba902b7,congo=t61rcWkgMzE", + TraceContext.vetTracestate(" rojo=00f067aa0ba902b7,congo=t61rcWkgMzE ")); + assertNull(TraceContext.vetTracestate("bad\nvalue")); + StringBuilder huge = new StringBuilder(); + while(huge.length() <= 512) { + huge.append("k=v,"); + } + assertNull(TraceContext.vetTracestate(huge.toString())); + assertNull(TraceContext.vetTracestate("")); + } + + @Test + @DisplayName("a tracestate that is not a valid list is discarded whole") + void tracestateGrammar() { + assertNull(TraceContext.vetTracestate("bad"), "a member with no key=value"); + assertNull(TraceContext.vetTracestate("a=1,a=2"), "a key twice"); + assertNull(TraceContext.vetTracestate("Upper=1"), "keys are lower case"); + assertNull(TraceContext.vetTracestate("a=x,b=has=equals")); + StringBuilder many = new StringBuilder(); + for(int i = 0 ; i < 33 ; i++) { + many.append(i == 0 ? "" : ",").append("k").append(i).append("=v"); + } + assertNull(TraceContext.vetTracestate(many.toString()), "33 members"); + assertEquals("tenant@sys=1,k/_-*=v v", TraceContext.vetTracestate("tenant@sys=1,k/_-*=v v")); + assertEquals("a=1,b=2", TraceContext.vetTracestate(" a=1, ,,b=2, "), + "empty members are accepted, as the specification requires, and not sent on"); + } + + @Test + @DisplayName("a sampler that takes no argument ignores one it was given") + void samplerArgumentOnlyForRatio() throws Exception { + // A shared template sets OTEL_TRACES_SAMPLER_ARG for whatever sampler it + // expects; a service that chose always_on must still start. + assertTrue(Sampler.parse("always_on", "not-a-number").sample(false, false, 1)); + assertFalse(Sampler.parse("parentbased_always_off", "x").sample(false, false, 1)); + assertRefused("parentbased_traceidratio", "x"); + } + + @Test + @DisplayName("a logged endpoint loses its userinfo and its query") + void endpointRedaction() { + assertEquals("https://@collector.example/v1/traces?", + BatchExporter.redact("https://user:secret@collector.example/v1/traces?token=t")); + assertEquals("http://collector.example:4318/v1/traces", + BatchExporter.redact("http://collector.example:4318/v1/traces")); + // A fragment carries a token as readily as a query does. + assertEquals("https://collector.example/v1/traces#", + BatchExporter.redact("https://collector.example/v1/traces#access_token=secret")); + assertEquals("https://c.example/v1#", + BatchExporter.redact("https://c.example/v1#frag?token=secret")); + // Through the LAST '@': the first is part of the password. + assertEquals("https://@host.example", + BatchExporter.redact("https://alice:secret@tenant@host.example")); + // And past a backslash, which some parsers keep in the authority. + assertEquals("https://@b/v1", + BatchExporter.redact("https://host.example\\a@b/v1")); + } + + @Test + @DisplayName("a relay token no header could carry is refused, without echoing it") + void unsendableRelayTokensAreRefused() throws Exception { + String[] bad = {"s3cret\n", "s3cret\r\n", " s3cret", "s3cret\t", "s3\u0001cret"}; + for(String token : bad) { + java.util.Properties settings = new java.util.Properties(); + settings.setProperty(OtlpTracer.ENDPOINT, "http://127.0.0.1:9"); + settings.setProperty(OtlpTracer.RELAY, "true"); + settings.setProperty(OtlpTracer.RELAY_TOKEN, token); + OtlpTracer tracer = new OtlpTracer(); + java.io.IOException refused = org.junit.jupiter.api.Assertions.assertThrows( + java.io.IOException.class, + () -> tracer.open(com.codename1.backend.Config.of(settings, "test"))); + assertTrue(refused.getMessage().contains(OtlpTracer.RELAY_TOKEN), refused.getMessage()); + assertFalse(refused.getMessage().contains("s3"), "the secret was echoed: " + + refused.getMessage()); + tracer.shutdown(0); + } + assertTrue(OtlpTracer.sendableFieldValue("s3cret with\tinner space")); + } + + @Test + @DisplayName("once shutdown begins the exporter takes nothing more") + void aStoppingExporterRefusesLateWork() throws Exception { + BatchExporter exporter = new BatchExporter("http://127.0.0.1:9/v1/traces", + new java.util.ArrayList(), true, new java.util.LinkedHashMap(), 16, 4, 60000, 1024); + exporter.start(); + exporter.shutdown(0); + // A span from a request still in flight when its tracer was replaced, and + // a relayed payload arriving just as late: both are dropped and counted, + // never handed to a worker that is supposed to be finishing. + exporter.add(null); + assertFalse(exporter.addRelayed(new byte[] {1}, "application/json"), + "a stopping exporter accepted a relayed payload"); + java.util.Map metrics = new java.util.LinkedHashMap(); + exporter.metrics(metrics); + assertEquals(Long.valueOf(1), metrics.get("spansDropped")); + assertEquals(Integer.valueOf(0), metrics.get("spansQueued")); + } + + @Test + @DisplayName("the traces path goes on the endpoint's path, not after its query") + void tracesPathBeforeQuery() { + assertEquals("https://c.example/otlp/v1/traces?api-key=s3cret", + OtlpTracer.appendTracesPath("https://c.example/otlp?api-key=s3cret")); + assertEquals("http://localhost:4318/v1/traces", + OtlpTracer.appendTracesPath("http://localhost:4318/")); + assertEquals("http://localhost:4318/v1/traces", + OtlpTracer.appendTracesPath("http://localhost:4318")); + } + + @Test + @DisplayName("relay input: whole base64 only, and integers in their range") + void relayValueValidation() throws Exception { + assertRelayRefuses("{\"bytesValue\":\"A\"}", "a lone base64 character is no byte"); + assertRelayRefuses("{\"bytesValue\":\"AA=garbage\"}", "data after padding"); + assertRelayRefuses("{\"bytesValue\":\"A=AA\"}", "padding in the middle"); + assertRelayRefuses("{\"intValue\":1e100}", "a number past 64 bits"); + assertRelayRefuses("{\"intValue\":\"9223372036854775808\"}", "one past Long.MAX_VALUE"); + relay("{\"bytesValue\":\"AAE=\"}"); + relay("{\"bytesValue\":\"AAE\"}"); + relay("{\"bytesValue\":\"-_8\"}"); + relay("{\"intValue\":\"-9223372036854775808\"}"); + } + + private static void relay(String anyValue) throws Exception { + OtlpSchema.protobuf(OtlpSchema.sanitize(com.codename1.backend.Json.parseObject( + "{\"resourceSpans\":[{\"scopeSpans\":[{\"spans\":[{\"traceId\":\"" + + "4bf92f3577b34da6a3ce929d0e0e4736\",\"spanId\":\"00f067aa0ba902b7\"," + + "\"attributes\":[{\"key\":\"k\",\"value\":" + anyValue + "}]}]}]}]}"))); + } + + private static void assertRelayRefuses(String anyValue, String why) { + try { + relay(anyValue); + } catch (Exception expected) { + return; + } + throw new AssertionError("accepted " + anyValue + ": " + why); + } + + @Test + @DisplayName("a partial success is read from either encoding") + void partialSuccessDecoding() throws Exception { + long[] rejected = new long[1]; + String[] message = new String[1]; + byte[] proto = io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse + .newBuilder() + .setPartialSuccess(io.opentelemetry.proto.collector.trace.v1 + .ExportTracePartialSuccess.newBuilder() + .setRejectedSpans(3).setErrorMessage("too old").build()) + .build().toByteArray(); + OtlpSchema.protobufPartialSuccess(proto, rejected, message); + assertEquals(3, rejected[0]); + assertEquals("too old", message[0]); + rejected[0] = 0; + message[0] = null; + OtlpSchema.jsonPartialSuccess( + "{\"partialSuccess\":{\"rejectedSpans\":\"5\",\"errorMessage\":\"bad\"}}", + rejected, message); + assertEquals(5, rejected[0]); + assertEquals("bad", message[0]); + } + + @Test + @DisplayName("the ratio sampler agrees with itself for one trace id") + void samplerIsDeterministicPerTrace() throws Exception { + Sampler half = Sampler.parse("traceidratio", "0.5"); + int sampled = 0; + for(long id = 1 ; id <= 10000 ; id++) { + long lo = id * 0x9E3779B97F4A7C15L; + boolean first = half.sample(false, false, lo); + assertEquals(first, half.sample(false, false, lo)); + if(first) { + sampled++; + } + } + assertTrue(sampled > 4500 && sampled < 5500, "sampled " + sampled + " of 10000"); + } + + @Test + @DisplayName("parent-based sampling follows the caller and ignores its own ratio") + void parentBased() throws Exception { + Sampler never = Sampler.parse("parentbased_always_off", null); + assertTrue(never.sample(true, true, 1)); + assertFalse(never.sample(true, false, 1)); + assertFalse(never.sample(false, false, 1)); + Sampler always = Sampler.parse(null, null); + assertFalse(always.sample(true, false, 1)); + assertTrue(always.sample(false, false, 1)); + } + + @Test + @DisplayName("an unknown sampler or a ratio out of range is refused, not defaulted") + void badSamplerConfiguration() { + assertRefused("jaeger_remote", null); + assertRefused("traceidratio", "1.5"); + assertRefused("traceidratio", "half"); + } + + private static void assertRefused(String name, String arg) { + try { + Sampler.parse(name, arg); + } catch (java.io.IOException expected) { + return; + } + throw new AssertionError("accepted sampler " + name + " / " + arg); + } + + @Test + @DisplayName("an X-Ray trace header becomes the equivalent traceparent") + void xray() throws Exception { + java.lang.reflect.Method convert = com.codename1.backend.Tracing.class + .getDeclaredMethod("fromXRay", String.class); + convert.setAccessible(true); + assertEquals("00-5759e988bd862e3fe1be46a994272793-53995c3f42cd8ad8-01", + convert.invoke(null, + "Root=1-5759e988-bd862e3fe1be46a994272793;Parent=53995c3f42cd8ad8;Sampled=1")); + assertEquals("00-5759e988bd862e3fe1be46a994272793-53995c3f42cd8ad8-00", + convert.invoke(null, + "Root=1-5759e988-bd862e3fe1be46a994272793;Parent=53995c3f42cd8ad8;Sampled=0")); + assertNull(convert.invoke(null, "Root=1-5759e988-bd862e3fe1be46a994272793")); + assertNull(convert.invoke(null, (Object)null)); + } + + @Test + @DisplayName("a trace endpoint must name a host, and a valid port if it names one") + void endpointAuthority() { + assertTrue(OtlpTracer.hasHttpAuthority("https://collector.example/v1/traces")); + assertTrue(OtlpTracer.hasHttpAuthority("http://user:pw@127.0.0.1:4318")); + assertTrue(OtlpTracer.hasHttpAuthority("http://[::1]:4318/v1/traces")); + assertTrue(OtlpTracer.hasHttpAuthority("HTTP://c.example:/v1/traces")); + assertFalse(OtlpTracer.hasHttpAuthority("https://")); + assertFalse(OtlpTracer.hasHttpAuthority("https:///v1/traces")); + assertFalse(OtlpTracer.hasHttpAuthority("http://user@:4318")); + assertFalse(OtlpTracer.hasHttpAuthority("http://c.example:70000")); + assertFalse(OtlpTracer.hasHttpAuthority("http://c.example:43x8")); + assertFalse(OtlpTracer.hasHttpAuthority("http://[::1")); + assertFalse(OtlpTracer.hasHttpAuthority("ftp://c.example")); + // A host no resolver or libcurl can use, though it is not empty. + assertFalse(OtlpTracer.hasHttpAuthority("https://collector example/v1/traces")); + assertFalse(OtlpTracer.hasHttpAuthority("https://collector\\example")); + assertFalse(OtlpTracer.hasHttpAuthority("https://collector\texample")); + assertFalse(OtlpTracer.hasHttpAuthority("https://[not-an-address]:4318")); + assertFalse(OtlpTracer.hasHttpAuthority("https://[1234]")); + assertFalse(OtlpTracer.hasHttpAuthority("https://host]:4318")); + assertTrue(OtlpTracer.hasHttpAuthority("https://10.0.0.7:4318")); + assertTrue(OtlpTracer.hasHttpAuthority("https://[::ffff:10.0.0.7]:4318")); + assertTrue(OtlpTracer.hasHttpAuthority("https://otel-collector.svc_1.local")); + // Userinfo is checked too, not skipped. + assertFalse(OtlpTracer.hasHttpAuthority("https://bad value@collector.example")); + assertFalse(OtlpTracer.hasHttpAuthority("https://u%zz@collector.example")); + assertFalse(OtlpTracer.hasHttpAuthority("https://a@b@collector.example")); + assertTrue(OtlpTracer.hasHttpAuthority("https://user:p%40ss@collector.example")); + // IPv6 by structure, not by its characters. + assertFalse(OtlpTracer.hasHttpAuthority("https://[:::]:4318")); + assertFalse(OtlpTracer.hasHttpAuthority("https://[1::2::3]:4318")); + assertFalse(OtlpTracer.hasHttpAuthority("https://[1:2:3:4:5:6:7:8:9]:4318")); + assertFalse(OtlpTracer.hasHttpAuthority("https://[::ffff:300.0.0.1]:4318")); + assertFalse(OtlpTracer.hasHttpAuthority("https://[1:2:3:4:5:6:7]:4318")); + assertFalse(OtlpTracer.hasHttpAuthority("https://[12345::1]:4318")); + assertTrue(OtlpTracer.hasHttpAuthority("https://[::1]:4318")); + assertTrue(OtlpTracer.hasHttpAuthority("https://[::ffff:10.0.0.7]:4318")); + assertTrue(OtlpTracer.hasHttpAuthority("https://[2001:db8::1]:4318")); + assertTrue(OtlpTracer.hasHttpAuthority("https://[1:2:3:4:5:6:7:8]:4318")); + assertTrue(OtlpTracer.hasHttpAuthority("https://[::]:4318")); + // A fragment is never sent, so a credential there never arrives. + assertFalse(OtlpTracer.hasHttpAuthority("https://collector.example/v1#api-key=s3cret")); + // The whole URL, not only its authority. + assertFalse(OtlpTracer.hasHttpAuthority("https://collector.example/bad value")); + assertFalse(OtlpTracer.hasHttpAuthority("https://collector.example/v1?q=a\tb")); + } + + @Test + @DisplayName("the relay ceiling counts UTF-8 bytes, not characters") + void relayCountsBytes() { + assertEquals(3, OtlpRelay.utf8Length("abc", 100)); + assertEquals(6, OtlpRelay.utf8Length("\u0800\u0800", 100), + "a three-byte character counted as one"); + assertEquals(4, OtlpRelay.utf8Length("\ud83d\ude00", 100), "a surrogate pair is 4 bytes"); + assertEquals(3, OtlpRelay.utf8Length("\ud83d", 100), "a lone surrogate encodes as U+FFFD"); + assertEquals(2, OtlpRelay.utf8Length("\u00e9", 100)); + assertTrue(OtlpRelay.utf8Length("\u0800\u0800\u0800\u0800", 5) > 5); + } + + @Test + @DisplayName("the relay path is refused unless matchable, and normalized as requests are") + void relayPath() { + assertEquals("/otel/v1/traces", OtlpTracer.canonicalPath("/otel/v1/traces")); + assertEquals("/t%C3%A9l%C3%A9metry", OtlpTracer.canonicalPath("/t%c3%a9l%c3%a9metry"), + "a kept escape gets upper-case hex, as the server spells it"); + assertEquals("/otel/traces", OtlpTracer.canonicalPath("/otel/%74races"), + "an escaped unreserved character is decoded, as the server decodes it"); + assertEquals("/a%2Fb", OtlpTracer.canonicalPath("/a%2fb")); + assertNull(OtlpTracer.canonicalPath("/t\u00e9l\u00e9metry"), "non-ASCII"); + assertNull(OtlpTracer.canonicalPath("/otel?x=1"), "a query never matches"); + assertNull(OtlpTracer.canonicalPath("/otel#f")); + assertNull(OtlpTracer.canonicalPath("/otel traces")); + assertNull(OtlpTracer.canonicalPath("/otel%2"), "a truncated escape"); + assertNull(OtlpTracer.canonicalPath("/otel%zz")); + assertNull(OtlpTracer.canonicalPath("otel")); + assertNull(OtlpTracer.canonicalPath("")); + } + + @Test + @DisplayName("a supplementary character survives percent-decoding beside an escape") + void percentDecodingKeepsSurrogatePairs() throws Exception { + java.util.Map out = new java.util.LinkedHashMap(); + OtlpTracer.parsePairs("label=\ud83d\ude00%20ok", out, "test"); + assertEquals("\ud83d\ude00 ok", out.get("label")); + } + + @Test + @DisplayName("a percent escape that is not well-formed UTF-8 is refused, not mangled") + void malformedUtf8IsRefused() throws Exception { + for(String bad : new String[] {"service.name=orders%C3%28", "x=%C0%AF", "x=%ED%A0%80"}) { + try { + OtlpTracer.parsePairs(bad, new java.util.LinkedHashMap(), "test"); + throw new AssertionError("accepted " + bad); + } catch (java.io.IOException expected) { + assertTrue(expected.getMessage().contains("UTF-8"), expected.getMessage()); + } + } + java.util.Map ok = new java.util.LinkedHashMap(); + OtlpTracer.parsePairs("service.name=caf%C3%A9", ok, "test"); + assertEquals("caf\u00e9", ok.get("service.name")); + } + + @Test + @DisplayName("a parent span id may be empty for a root, never all zeros") + void zeroParentIsRefused() throws Exception { + String span = "{\"resourceSpans\":[{\"scopeSpans\":[{\"spans\":[{\"traceId\":" + + "\"4bf92f3577b34da6a3ce929d0e0e4736\",\"spanId\":\"00f067aa0ba902b7\"," + + "\"parentSpanId\":\"PARENT\",\"name\":\"x\"}]}]}]}"; + OtlpSchema.sanitize((java.util.Map)com.codename1.backend.Json.parse(span.replace("PARENT", ""))); + OtlpSchema.sanitize((java.util.Map)com.codename1.backend.Json.parse( + span.replace("PARENT", "b7ad6b7169203331"))); + java.io.IOException refused = org.junit.jupiter.api.Assertions.assertThrows( + java.io.IOException.class, () -> OtlpSchema.sanitize((java.util.Map) + com.codename1.backend.Json.parse(span.replace("PARENT", "0000000000000000")))); + assertTrue(refused.getMessage().contains("parentSpanId"), refused.getMessage()); + } + + @Test + @DisplayName("a span's name is bounded like its attributes") + void spanNamesAreBounded() throws Exception { + java.util.Properties settings = new java.util.Properties(); + settings.setProperty(OtlpTracer.ENDPOINT, "http://127.0.0.1:9"); + OtlpTracer tracer = new OtlpTracer(); + assertTrue(tracer.open(com.codename1.backend.Config.of(settings, "test"))); + try { + StringBuilder huge = new StringBuilder(); + while(huge.length() < 50000) { + huge.append("name "); + } + com.codename1.backend.Span span = tracer.startSpan(huge.toString(), + com.codename1.backend.Span.KIND_INTERNAL, null, null, null); + assertTrue(span.getName().length() <= OtelSpan.MAX_VALUE_LENGTH); + span.updateName(huge.toString()); + assertTrue(span.getName().length() <= OtelSpan.MAX_VALUE_LENGTH); + } finally { + tracer.shutdown(0); + } + } + + @Test + @DisplayName("an oversized attribute key is dropped, never cut into another key") + void oversizedKeysAreDropped() throws Exception { + java.util.Properties settings = new java.util.Properties(); + settings.setProperty(OtlpTracer.ENDPOINT, "http://127.0.0.1:9"); + OtlpTracer tracer = new OtlpTracer(); + assertTrue(tracer.open(com.codename1.backend.Config.of(settings, "test"))); + try { + StringBuilder huge = new StringBuilder(); + while(huge.length() <= OtelSpan.MAX_KEY_LENGTH) { + huge.append("key."); + } + OtelSpan span = (OtelSpan) tracer.startSpan("keys", + com.codename1.backend.Span.KIND_INTERNAL, null, null, null); + span.setAttribute(huge.toString(), "a"); + span.setAttribute(huge.toString() + "other", 1L); + span.setAttribute("short", true); + assertEquals(1, span.attributes.size(), "an oversized key was kept: " + span.attributes); + assertEquals(2, span.droppedAttributes); + } finally { + tracer.shutdown(0); + } + } + + @Test + @DisplayName("one observed span id does not predict the next trace id") + void idsAreNotPredictableFromOneAnother() throws Exception { + java.util.Properties settings = new java.util.Properties(); + settings.setProperty(OtlpTracer.ENDPOINT, "http://127.0.0.1:9"); + OtlpTracer tracer = new OtlpTracer(); + assertTrue(tracer.open(com.codename1.backend.Config.of(settings, "test"))); + try { + // The attack the old generator allowed: SplitMix64 is invertible, so a + // span id seen in a header gives back the counter, and one step on from + // it is the next root span's trace id. Run it against each pair. + java.math.BigInteger mod = java.math.BigInteger.ONE.shiftLeft(64); + long inv1 = java.math.BigInteger.valueOf(0xBF58476D1CE4E5B9L).mod(mod) + .modInverse(mod).longValue(); + long inv2 = java.math.BigInteger.valueOf(0x94D049BB133111EBL).mod(mod) + .modInverse(mod).longValue(); + int predicted = 0; + for(int iter = 0 ; iter < 200 ; iter++) { + OtelSpan seen = (OtelSpan) tracer.startSpan("seen", + com.codename1.backend.Span.KIND_INTERNAL, null, null, null); + OtelSpan next = (OtelSpan) tracer.startSpan("next", + com.codename1.backend.Span.KIND_INTERNAL, null, null, null); + long z = seen.spanId; + z = z ^ (z >>> 31) ^ (z >>> 62); + z *= inv2; + z = z ^ (z >>> 27) ^ (z >>> 54); + z *= inv1; + z = z ^ (z >>> 30) ^ (z >>> 60); + long state = z + 0x9E3779B97F4A7C15L; + long guess = state; + guess = (guess ^ (guess >>> 30)) * 0xBF58476D1CE4E5B9L; + guess = (guess ^ (guess >>> 27)) * 0x94D049BB133111EBL; + guess = guess ^ (guess >>> 31); + if(guess == next.traceHi) { + predicted++; + } + } + assertEquals(0, predicted, "trace ids followed from the previous span id"); + } finally { + tracer.shutdown(0); + } + } + + @Test + @DisplayName("a truncated value never ends in half a character") + void truncationKeepsPairsWhole() { + StringBuilder text = new StringBuilder(); + for(int i = 0 ; i < OtelSpan.MAX_VALUE_LENGTH - 1 ; i++) { + text.append('a'); + } + text.append("\ud83d\ude00tail"); + String bounded = OtelSpan.bound(text.toString()); + assertEquals(OtelSpan.MAX_VALUE_LENGTH - 1, bounded.length()); + assertFalse(Character.isHighSurrogate(bounded.charAt(bounded.length() - 1))); + } + + @Test + @DisplayName("the relay's CORS origin is * or one serialized origin") + void corsOrigin() throws Exception { + assertEquals("https://app.example.com", cors("https://app.example.com")); + assertEquals("http://localhost:8080", cors("http://localhost:8080")); + assertEquals("*", cors("*")); + // As the browser serializes it: lower case, no default port. + assertEquals("https://app.example.com", cors("HTTPS://APP.Example.COM:443")); + assertEquals("http://app.example.com", cors("http://app.example.com:80")); + assertEquals("https://app.example.com:8443", cors("https://app.example.com:08443")); + assertEquals("http://[::1]:8080", cors("http://[::1]:8080")); + assertNull(cors("")); + // Refused without quoting what made it unsafe. + try { + cors("https://user:p4ssword@app.example.com?token=s3cret"); + throw new AssertionError("accepted userinfo"); + } catch (java.io.IOException expected) { + assertFalse(expected.getMessage().contains("p4ssword"), expected.getMessage()); + assertFalse(expected.getMessage().contains("s3cret"), expected.getMessage()); + } + for(String bad : new String[] {"https://app.example.com/", "https://app.example.com/app", + "https://a.example, https://b.example", "app.example.com", "https://u@app.example", + "https://app.example.com?x=1"}) { + try { + cors(bad); + throw new AssertionError("accepted " + bad); + } catch (java.io.IOException expected) { + assertTrue(expected.getMessage().contains(OtlpTracer.RELAY_CORS_ORIGIN)); + } + } + } + + private static String cors(String value) throws java.io.IOException { + java.util.Properties settings = new java.util.Properties(); + settings.setProperty(OtlpTracer.RELAY_CORS_ORIGIN, value); + return OtlpTracer.corsOrigin(com.codename1.backend.Config.of(settings, "test")); + } + + @Test + @DisplayName("a collector's error text is bounded before it is kept or logged") + void diagnosticsAreBounded() { + StringBuilder huge = new StringBuilder(); + while(huge.length() < 100000) { + huge.append("rejected "); + } + String kept = BatchExporter.bounded(huge.toString()); + assertTrue(kept.length() < BatchExporter.MAX_ERROR_CHARS + 32, "kept " + kept.length()); + assertEquals("short", BatchExporter.bounded("short")); + } +} diff --git a/maven/cn1-hardening/src/main/java/com/codename1/hardening/BuiltinKeepRules.java b/maven/cn1-hardening/src/main/java/com/codename1/hardening/BuiltinKeepRules.java index e680837e0d9..6d3724dc60b 100644 --- a/maven/cn1-hardening/src/main/java/com/codename1/hardening/BuiltinKeepRules.java +++ b/maven/cn1-hardening/src/main/java/com/codename1/hardening/BuiltinKeepRules.java @@ -37,11 +37,11 @@ */ public final class BuiltinKeepRules { - /** The seven generated bootstrap classes the builders splice into the app stub. */ + /** The generated bootstrap classes the builders splice into the app stub. */ private static final String[] BOOTSTRAPS = { "MapperBootstrap", "BinderBootstrap", "DaoBootstrap", "RestClientBootstrap", "ProtoBootstrap", "GrpcClientBootstrap", "GraphQLClientBootstrap", - "IntentBootstrap" + "IntentBootstrap", "TelemetryBootstrap" }; private BuiltinKeepRules() { diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java index 595722f4fa6..e10b8c316b0 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java @@ -4960,14 +4960,7 @@ protected Properties getLocalBuilderProperties() { /// `sourceZip` passed to `build(...)`) contains the build-time /// generated `com.codename1.router.generated.Routes` class. protected static boolean projectHasRouteDispatcher(File sourceZip) { - if (sourceZip == null || !sourceZip.isFile()) { - return false; - } - try (java.util.zip.ZipFile zf = new java.util.zip.ZipFile(sourceZip)) { - return zf.getEntry("com/codename1/router/generated/Routes.class") != null; - } catch (IOException e) { - return false; - } + return projectHasBootstrap(sourceZip, "com/codename1/router/generated/Routes.class"); } /// Stub-source fragment to splice into a generated application stub @@ -5031,6 +5024,13 @@ protected static String annotationFrameworksInstallSource(File sourceZip, String if (projectHasBootstrap(sourceZip, "cn1app/IntentBootstrap.class")) { sb.append(indent).append("new cn1app.IntentBootstrap();\n"); } + // @OpenTelemetry: installs com.codename1.telemetry.Telemetry, which puts a + // tracer on NetworkManager. Before Display.init like the rest, so the + // first request the app makes is already traced; the install reads nothing + // from the display until its first export. + if (projectHasBootstrap(sourceZip, "cn1app/TelemetryBootstrap.class")) { + sb.append(indent).append("new cn1app.TelemetryBootstrap();\n"); + } return sb.toString(); } @@ -5038,7 +5038,16 @@ protected static String annotationFrameworksInstallSource(File sourceZip, String /// `jar-with-dependencies`) contains `entryPath`. Used to gate the /// per-feature bootstrap install lines so projects that don't use /// every annotation framework still produce a clean stub. + /// + /// `sourceZip` may also be the DIRECTORY the project's classes were unpacked + /// into. The builders that write their stub after unpacking -- JavaScript and + /// the native desktop targets -- hold that directory rather than the jar, and + /// with only a jar to ask they emitted no install lines at all: every + /// annotation framework was silently inert on those platforms. protected static boolean projectHasBootstrap(File sourceZip, String entryPath) { + if (sourceZip != null && sourceZip.isDirectory()) { + return new File(sourceZip, entryPath.replace('/', File.separatorChar)).isFile(); + } if (sourceZip == null || !sourceZip.isFile()) { return false; } diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/JavaScriptBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/JavaScriptBuilder.java index ed840b78589..c8f6930fc7a 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/JavaScriptBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/JavaScriptBuilder.java @@ -171,6 +171,12 @@ public boolean build(File sourceZip, BuildRequest request) throws BuildException File stageClasses = new File(buildDir, "stage-classes"); File portClasses = new File(buildDir, "port-classes"); File translatorOut = new File(buildDir, "translator-output"); + // The staged classes are emptied first: the build directory is stable + // across builds and unzip() only overwrites what the jar carries, so a + // class deleted since the last build -- including a generated bootstrap + // whose annotation was removed, which annotationFrameworksInstallSource() + // probes this directory for -- would otherwise still be installed. + MacOSNativeBuilder.deleteRecursively(stageClasses); stageClasses.mkdirs(); portClasses.mkdirs(); translatorOut.mkdirs(); @@ -539,6 +545,14 @@ private File writeLauncher(File workDir, String launcherName, String packageName // runnable at exactly that point; a post-bootstrap stamp would miss startup because // bootstrap runs init/start inline. hardeningRuntimeProperties emits // Display.getInstance().setProperty(...) lines. + // The generated @Route dispatcher and annotation-framework bootstraps + // (@Mapped, @Bindable, @Entity, the REST/gRPC/GraphQL clients, intents, + // @OpenTelemetry), before Display.init as every other port's stub does. + // Without these lines each of those features compiled into the web build + // and silently did nothing. Direct references, so the translator keeps + // the generated classes. + pw.print(routeDispatcherInstallSource(stageClasses, " ")); + pw.print(annotationFrameworksInstallSource(stageClasses, " ")); pw.println(" ParparVMBootstrap.bootstrap(new " + mainClass + "(), new Runnable() {"); pw.println(" public void run() {"); pw.print(hardeningRuntimeProperties(request)); diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/LinuxNativeBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/LinuxNativeBuilder.java index 0ee6b97cc2a..a575de0f1a8 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/LinuxNativeBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/LinuxNativeBuilder.java @@ -200,6 +200,19 @@ public boolean build(File sourceZip, BuildRequest request) throws BuildException File classesDir = new File(tmpFile, "classes"); File resDir = new File(tmpFile, "res"); File buildinRes = new File(tmpFile, "btres"); + // Emptied, not merely created, for the reason MacOSNativeBuilder gives: + // these paths are stable across builds and unzip() only overwrites what + // the archive carries, so a class deleted since the last build would still + // be translated -- and a removed @OpenTelemetry or @Route would leave its + // generated bootstrap behind, which annotationFrameworksInstallSource() + // probes this directory for and would keep installing. + try { + MacOSNativeBuilder.deleteRecursively(classesDir); + MacOSNativeBuilder.deleteRecursively(resDir); + MacOSNativeBuilder.deleteRecursively(buildinRes); + } catch (IOException ex) { + throw new BuildException("Failed to clear the staged build inputs", ex); + } classesDir.mkdirs(); resDir.mkdirs(); buildinRes.mkdirs(); @@ -675,6 +688,12 @@ private void writeBootstrapStub(BuildRequest request, File classesDir, File stub src.append(" public static void main(String[] argv) {\n"); src.append(registerNatives); src.append(" final ").append(main).append(" app = new ").append(main).append("();\n"); + // The generated @Route dispatcher and annotation-framework bootstraps, before + // Display.init as the iOS and Android stubs install them. This stub used to + // install neither, so @Route, @Mapped, the generated REST/gRPC/GraphQL + // clients and @OpenTelemetry all compiled here and did nothing at run time. + src.append(routeDispatcherInstallSource(classesDir, " ")); + src.append(annotationFrameworksInstallSource(classesDir, " ")); src.append(" Display.init(null);\n"); // The application's identity, which nothing else gives this platform. The stub passes // null to Display.init, so the implementation never derives a package from an object, and diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacOSNativeBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacOSNativeBuilder.java index 3df8647e6b2..cbf533bacc6 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacOSNativeBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacOSNativeBuilder.java @@ -871,7 +871,12 @@ void writeStub(BuildRequest request, File stubSource, File classesDir, // this call; this builder was written without it. + " com.codename1.impl.ios.IOSImplementation.setIosMode(\"" + themeMode + "\");\n" - + routeDispatcherInstallSource(null, " ") + // The unpacked classes, not a jar: this builder never had one to + // pass, and passing null made both of these answer "none" for + // every project -- @Route and every annotation framework were + // inert on macOS. + + routeDispatcherInstallSource(classesDir, " ") + + annotationFrameworksInstallSource(classesDir, " ") + " Display.init(stub);\n" + " }\n" + "}\n"; diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/WindowsNativeBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/WindowsNativeBuilder.java index cf5de52a3d3..bc8e9c509b8 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/WindowsNativeBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/WindowsNativeBuilder.java @@ -182,6 +182,19 @@ public boolean build(File sourceZip, BuildRequest request) throws BuildException File classesDir = new File(tmpFile, "classes"); File resDir = new File(tmpFile, "res"); File buildinRes = new File(tmpFile, "btres"); + // Emptied, not merely created, for the reason MacOSNativeBuilder gives: + // these paths are stable across builds and unzip() only overwrites what + // the archive carries, so a class deleted since the last build would still + // be translated -- and a removed @OpenTelemetry or @Route would leave its + // generated bootstrap behind, which annotationFrameworksInstallSource() + // probes this directory for and would keep installing. + try { + MacOSNativeBuilder.deleteRecursively(classesDir); + MacOSNativeBuilder.deleteRecursively(resDir); + MacOSNativeBuilder.deleteRecursively(buildinRes); + } catch (IOException ex) { + throw new BuildException("Failed to clear the staged build inputs", ex); + } classesDir.mkdirs(); resDir.mkdirs(); buildinRes.mkdirs(); @@ -1231,6 +1244,12 @@ private void writeBootstrapStub(BuildRequest request, File classesDir, File stub src.append(" public static void main(String[] argv) {\n"); src.append(registerNatives); src.append(" final ").append(main).append(" app = new ").append(main).append("();\n"); + // The generated @Route dispatcher and annotation-framework bootstraps, before + // Display.init as the iOS and Android stubs install them. This stub used to + // install neither, so @Route, @Mapped, the generated REST/gRPC/GraphQL + // clients and @OpenTelemetry all compiled here and did nothing at run time. + src.append(routeDispatcherInstallSource(classesDir, " ")); + src.append(annotationFrameworksInstallSource(classesDir, " ")); src.append(" Display.init(null);\n"); // The application's identity, which nothing else gives this platform. The stub passes // null to Display.init, so the implementation never derives a package from an object, and diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java index aaa848a438c..8d1e2d4da56 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java @@ -82,6 +82,12 @@ public final class RestControllerAnnotationProcessor extends AbstractAnnotationP private static final String RESPONSE_STATUS = PKG + "ResponseStatus;"; private static final String WEBSOCKET_MAPPING = PKG + "WebSocketMapping;"; private static final String WEBSOCKET_INTERFACE = "com/codename1/backend/WebSocket"; + static final String OPEN_TELEMETRY = PKG + "OpenTelemetry;"; + + /// The application.properties key that enables tracing without touching the + /// source, read at build time from the module directory -- where Config reads + /// the same file at run time. + static final String OTEL_ENABLED_PROPERTY = "cn1.otel.enabled"; /** Mapping annotation to the HTTP method it stands for. */ private static final Map MAPPINGS; @@ -175,6 +181,12 @@ private static final class WebSocketEndpoint { /// when the entry point is written; see [#hasGeneratedDaos]. private boolean daos; + /// Whether the generated entry point installs a tracer, and the service + /// name it passes. Settled in [#finish], before any source is generated, + /// because the routers name their routes for it too. + boolean telemetry; + String telemetryServiceName; + private final Map routeShapes = new LinkedHashMap(); /** Which controller claimed each shape, so a clash names the other one. */ @@ -1269,6 +1281,10 @@ public void finish(ProcessorContext ctx) throws ProcessingException { } return; } + resolveTelemetry(ctx); + if (ctx.hasErrors()) { + return; + } Map sources = new LinkedHashMap(); for (Controller c : controllers.values()) { String router = qualify(c.packageName, c.routerSimpleName); @@ -1463,6 +1479,16 @@ private static void emitRoute(StringBuilder sb, Route route, int index, Controll sb.append(pad).append("if (bound != null) {\n"); pad = " "; } + // The TEMPLATE names the span, never the path: /pets/7 and /pets/8 are + // one operation, and a trace backend grouping by path would give every id + // its own. Emitted in EVERY build, not only a traced one: a program can + // install a tracer itself (Tracing.install(OtlpTracer.open(...))), and a + // router that never named its routes merged them all under "GET". It + // costs one static read with no tracer -- Tracing is linked by HttpServer + // regardless -- and it is the OTLP tracer, not this hook, that a build + // without @OpenTelemetry leaves out. + sb.append(pad).append("com.codename1.backend.Tracing.route(") + .append(quote(route.pattern)).append(");\n"); emitRequiredGuards(sb, route, pad); emitScalarGuards(sb, route, pad); @@ -2501,6 +2527,15 @@ String generateBootstrap(String packageName) { // left here is the part that differs between one server and the next: // which controllers there are and what each of them is given. sb.append(" com.codename1.backend.Backend.builder()\n"); + if (telemetry) { + // The ONLY reference to the tracer implementation anywhere in the + // program, which is what keeps it out of a binary that does not ask + // for it: the translator drops what nothing reaches. + sb.append(" .tracing(new com.codename1.backend.otel.OtlpTracer(") + .append(telemetryServiceName == null || telemetryServiceName.length() == 0 + ? "null" : quote(telemetryServiceName)) + .append("))\n"); + } if (needsDatabase() || needsDatabaseForWebSockets()) { // A controller that declares a DataSource or an EntityManager needs // a database, and this is where the build says so: the builder opens @@ -2553,6 +2588,126 @@ String generateBootstrap(String packageName) { return sb.toString(); } + /// Whether this module asked for tracing, and under what service name. + /// + /// `@OpenTelemetry` on any class with a source in this module, or + /// `cn1.otel.enabled` in the module's `application.properties`. The + /// annotation is looked for across the whole class index rather than only on + /// controllers, so it can sit on whichever class a project keeps its settings + /// on. Two annotations naming different services is refused: one server + /// reports as one service, and picking either silently would be a guess. + void resolveTelemetry(ProcessorContext ctx) { + telemetry = false; + telemetryServiceName = null; + String owner = null; + for (AnnotatedClass cls : ctx.getClassIndex().values()) { + AnnotationValues otel = cls.getClassAnnotation(OPEN_TELEMETRY); + if (otel == null) { + continue; + } + // The same orphan rule the controllers get: a class whose source was + // deleted left its .class behind, and must not keep tracing on. + if (!BuildHintAnnotationProcessor.hasBackingSource(cls, ctx.getCompileSourceRoots(), + ctx.getSourceEncoding())) { + continue; + } + // Trimmed: " " is no name at all, and kept raw it reached the entry + // point and became an empty service.name instead of unknown_service. + String name = otel.getStringOrDefault("serviceName", "").trim(); + if (telemetry && name.length() > 0 && telemetryServiceName != null + && telemetryServiceName.length() > 0 && !name.equals(telemetryServiceName)) { + ctx.error(cls, "@OpenTelemetry names the service \"" + name + "\" here and \"" + + telemetryServiceName + "\" on " + owner + ". A server reports as one " + + "service; keep one serviceName, or set OTEL_SERVICE_NAME instead."); + return; + } + telemetry = true; + if (name.length() > 0) { + telemetryServiceName = name; + owner = cls.getBinaryName(); + } + } + if (!telemetry && propertyEnablesTelemetry(ctx)) { + telemetry = true; + } + } + + /// `cn1.otel.enabled` from `application.properties` beside the module, the + /// file Config reads at run time. Only a literal truth value counts: a + /// `${...}` reference cannot be resolved at build time, and building the + /// tracer in on a guess would make the switch mean nothing. + private static boolean propertyEnablesTelemetry(ProcessorContext ctx) { + File file = applicationProperties(ctx); + if (file == null) { + return false; + } + java.util.Properties props = new java.util.Properties(); + InputStream in = null; + try { + in = new java.io.FileInputStream(file); + props.load(in); + } catch (IOException err) { + ctx.getLog().warn("cn1: could not read " + file + ": " + err.getMessage()); + return false; + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException ignored) { + // Closing a file that was only read cannot lose anything. + } + } + } + String value = props.getProperty(OTEL_ENABLED_PROPERTY); + if (value == null) { + return false; + } + String v = value.trim(); + return "true".equalsIgnoreCase(v) || "yes".equalsIgnoreCase(v) + || "on".equalsIgnoreCase(v) || "1".equals(v); + } + + /// The backend module's `application.properties`, or null. + /// + /// Looked for beside the module that owns the output directory FIRST + /// (`/target/classes`), because that is the file Config reads when the + /// module is run -- and the context's project directory is not reliably that + /// module: the annotation goal resolves it as the Codename One project, which + /// in a multi-module application is `common`. The packaging goal does pass the + /// module itself, which is the fallback. + private static File applicationProperties(ProcessorContext ctx) { + // Where a project keeps it, in the order Maven would see it. The copy in + // the classes directory FIRST: the documented layout is + // src/main/resources/application.properties, which process-resources has + // already copied there by the time this runs -- and looking only beside + // the module missed it, so cn1.otel.enabled=true in the standard place + // never linked the tracer. The source copy covers a build that skipped + // the resources phase; the module and project directories, the other + // places Config reads it from at run time. + List candidates = new ArrayList(); + File classes = ctx.getOutputClassDir(); + if (classes != null) { + candidates.add(new File(classes, "application.properties")); + if (classes.getParentFile() != null + && "target".equals(classes.getParentFile().getName())) { + File module = classes.getParentFile().getParentFile(); + candidates.add(new File(module, "src/main/resources/application.properties")); + candidates.add(new File(module, "application.properties")); + } + } + File dir = ctx.getProjectDir(); + if (dir != null) { + candidates.add(new File(dir, "src/main/resources/application.properties")); + candidates.add(new File(dir, "application.properties")); + } + for (File file : candidates) { + if (file.isFile()) { + return file; + } + } + return null; + } + /// Whether any controller declared a constructor that needs a database. private boolean needsDatabase() { for (Controller c : controllers.values()) { diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/TelemetryAnnotationProcessor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/TelemetryAnnotationProcessor.java new file mode 100644 index 00000000000..961b5fabf6c --- /dev/null +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/TelemetryAnnotationProcessor.java @@ -0,0 +1,548 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven.processors; + +import com.codename1.maven.annotations.AbstractAnnotationProcessor; +import com.codename1.maven.annotations.AnnotatedClass; +import com.codename1.maven.annotations.AnnotationValues; +import com.codename1.maven.annotations.JavaSourceCompiler; +import com.codename1.maven.annotations.ProcessingException; +import com.codename1.maven.annotations.ProcessorContext; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/// Build-time `@OpenTelemetry` processor for the app. +/// +/// Emits a single `cn1app.TelemetryBootstrap` whose constructor installs +/// `com.codename1.telemetry.Telemetry` with the annotation's settings. The per-build +/// application stub instantiates it before `Display.init` (see +/// `Executor.annotationFrameworksInstallSource`), and the JavaSE port finds it by +/// name for the simulator. That constructor is the ONLY reference to the telemetry +/// package, which is how an app that does not enable it ends up without it: the +/// translator and R8 both keep only what is reachable. +/// +/// Validated here rather than at run time, because the mistakes are in source the +/// build can read: no endpoint at all, both a relay and a direct endpoint, a header +/// that is not `Name: value`, a ratio outside 0..1. +public final class TelemetryAnnotationProcessor extends AbstractAnnotationProcessor { + + public static final String OPEN_TELEMETRY_DESC = "Lcom/codename1/annotations/OpenTelemetry;"; + + static final String BOOTSTRAP_BINARY = "cn1app.TelemetryBootstrap"; + static final String BOOTSTRAP_SIMPLE = "TelemetryBootstrap"; + + /// The one accepted annotation, and the class it was on. + private AnnotationValues accepted; + private String owner; + + @Override + public Set getAnnotationDescriptors() { + return Collections.singleton(OPEN_TELEMETRY_DESC); + } + + @Override + public void start(ProcessorContext ctx) throws ProcessingException { + accepted = null; + owner = null; + } + + @Override + public void processClass(AnnotatedClass cls, ProcessorContext ctx) throws ProcessingException { + AnnotationValues otel = cls.getClassAnnotation(OPEN_TELEMETRY_DESC); + if (otel == null || cls.isSynthetic()) { + return; + } + // A deleted class whose .class is still in target/classes must not keep + // telemetry on: the same orphan rule every generator here follows. + if (!BuildHintAnnotationProcessor.hasBackingSource(cls, ctx.getCompileSourceRoots(), + ctx.getSourceEncoding())) { + return; + } + if (accepted != null) { + ctx.error(cls, "@OpenTelemetry is on both " + owner + " and " + cls.getBinaryName() + + ". An app installs telemetry once; keep it on the main class."); + return; + } + String relay = otel.getStringOrDefault("relay", "").trim(); + String endpoint = otel.getStringOrDefault("endpoint", "").trim(); + if (relay.length() == 0 && endpoint.length() == 0) { + ctx.error(cls, "@OpenTelemetry on " + cls.getBinaryName() + " names neither a relay " + + "nor an endpoint, so there is nowhere to send spans. Set relay to the " + + "app's Codename One backend, or endpoint to an OTLP/HTTP collector."); + return; + } + if (relay.length() > 0 && endpoint.length() > 0) { + ctx.error(cls, "@OpenTelemetry on " + cls.getBinaryName() + " sets both relay and " + + "endpoint. Spans go one way: through the backend (relay) or straight " + + "to a collector (endpoint)."); + return; + } + if (!isHttpUrl(relay.length() > 0 ? relay : endpoint)) { + ctx.error(cls, "@OpenTelemetry on " + cls.getBinaryName() + " must name an http or " + + "https URL"); + return; + } + int position = 0; + for (String header : strings(otel.get("headers"))) { + position++; + int colon = header.indexOf(':'); + if (colon <= 0 || colon == header.length() - 1) { + // By position, never quoted: a malformed entry is still usually a + // credential ("Authorization Bearer ..."), and this message goes to + // compiler and CI logs. The later checks name only the header. + ctx.error(cls, "@OpenTelemetry header #" + position + " is not \"Name: value\" " + + "(its value is not shown, since it is usually a credential)"); + return; + } + // The whole field, not just the colon. The export is fail-silent by + // design, so a header the platform refuses -- a name that is not an + // HTTP token -- loses every span with nothing said, and a control + // character in the value is a header injection on a lenient transport. + // This is the one place the mistake is visible. + String name = header.substring(0, colon).trim(); + String value = header.substring(colon + 1).trim(); + if (!isToken(name)) { + ctx.error(cls, "@OpenTelemetry header name \"" + name + "\" is not a valid HTTP " + + "header name (letters, digits and !#$%&'*+-.^_`|~ only)"); + return; + } + // The ones TelemetryConfig refuses at run time, where the generated + // bootstrap would throw before Display.init. + if (name.equalsIgnoreCase("Content-Type") || name.equalsIgnoreCase("Content-Length") + || name.equalsIgnoreCase("Host") || name.equalsIgnoreCase("Transfer-Encoding")) { + ctx.error(cls, "@OpenTelemetry header " + name + " is set by the exporter from " + + "what it sends and cannot be configured"); + return; + } + if (value.length() == 0 || hasControl(value)) { + ctx.error(cls, "@OpenTelemetry header " + name + " has an empty value or one " + + "containing a control character such as a line break"); + return; + } + } + if (relay.length() > 0 && !strings(otel.get("headers")).isEmpty()) { + ctx.error(cls, "@OpenTelemetry on " + cls.getBinaryName() + " sets headers for a " + + "relay. The relay holds the collector's credentials; the app sends only " + + "relayToken. Move the headers to the backend's cn1.otel.headers."); + return; + } + // TelemetryConfig.relayToken refuses a control character at run time, where + // the generated bootstrap would throw before Display.init; refused here + // instead, where the build can say so. + String relayToken = otel.getStringOrDefault("relayToken", ""); + if (hasControl(relayToken)) { + ctx.error(cls, "@OpenTelemetry relayToken contains a control character such as a " + + "line break; it is sent as a header and no header may carry one"); + return; + } + // Refused at run time too: the backend trims the whitespace around a header + // value and compares the token exactly, so this one could never match. + if (relayToken.length() > 0 && (isBlank(relayToken.charAt(0)) + || isBlank(relayToken.charAt(relayToken.length() - 1)))) { + ctx.error(cls, "@OpenTelemetry relayToken begins or ends with whitespace, which " + + "a header cannot carry"); + return; + } + double ratio = ratio(otel.get("sampleRatio")); + if (!(ratio >= 0 && ratio <= 1)) { + ctx.error(cls, "@OpenTelemetry sampleRatio must be between 0 and 1, not " + ratio); + return; + } + accepted = otel; + owner = cls.getBinaryName(); + } + + @Override + public void finish(ProcessorContext ctx) throws ProcessingException { + if (ctx.hasErrors()) { + return; + } + if (accepted == null) { + // NOTHING ASKS FOR TELEMETRY NOW, so a bootstrap an earlier build left in + // target/classes has to go. The builders install whatever bootstrap class + // is present rather than asking whether the annotation still is, and + // Maven keeps target/classes across a build without clean -- so removing + // @OpenTelemetry, or deleting the class it was on, went on shipping the + // old endpoint, token and consent setting until someone ran clean. + File stale = new File(ctx.getOutputClassDir(), + BOOTSTRAP_BINARY.replace('.', File.separatorChar) + ".class"); + if (stale.isFile() && !stale.delete()) { + ctx.getLog().warn("cn1: could not remove the stale " + stale + "; a build " + + "without clean may still install telemetry the project no longer asks for"); + } + return; + } + Map sources = new LinkedHashMap(); + sources.put(BOOTSTRAP_BINARY, generateBootstrapSource(accepted)); + try { + List cp = new ArrayList(); + cp.add(ctx.getOutputClassDir()); + for (String element : ctx.getCompileClasspath()) { + cp.add(new File(element)); + } + JavaSourceCompiler.compile(sources, ctx.getOutputClassDir(), cp); + } catch (IOException ioe) { + throw new ProcessingException("Could not compile the generated telemetry bootstrap: " + + ioe.getMessage(), ioe); + } + ctx.getLog().info("cn1: generated " + BOOTSTRAP_BINARY + " from @OpenTelemetry on " + owner); + } + + /// The bootstrap's source. Package-visible so a test can read it. + static String generateBootstrapSource(AnnotationValues otel) { + String relay = otel.getStringOrDefault("relay", "").trim(); + String endpoint = otel.getStringOrDefault("endpoint", "").trim(); + StringBuilder sb = new StringBuilder(1024); + sb.append("package cn1app;\n\n"); + sb.append("// Auto-generated by cn1:process-annotations from @OpenTelemetry. Do not edit.\n"); + sb.append("@SuppressWarnings({\"all\"})\n"); + sb.append("public final class ").append(BOOTSTRAP_SIMPLE).append(" {\n"); + sb.append(" public ").append(BOOTSTRAP_SIMPLE).append("() {\n"); + sb.append(" com.codename1.telemetry.Telemetry.install(new com.codename1.telemetry.TelemetryConfig()\n"); + if (relay.length() > 0) { + sb.append(" .relay(").append(quote(relay)).append(")\n"); + } else { + sb.append(" .direct(").append(quote(endpoint)).append(")\n"); + } + String service = otel.getStringOrDefault("serviceName", "").trim(); + if (service.length() > 0) { + sb.append(" .serviceName(").append(quote(service)).append(")\n"); + } + for (String header : strings(otel.get("headers"))) { + int colon = header.indexOf(':'); + sb.append(" .header(").append(quote(header.substring(0, colon).trim())) + .append(", ").append(quote(header.substring(colon + 1).trim())).append(")\n"); + } + String token = otel.getStringOrDefault("relayToken", ""); + if (token.length() > 0) { + sb.append(" .relayToken(").append(quote(token)).append(")\n"); + } + double ratio = ratio(otel.get("sampleRatio")); + if (ratio < 1) { + sb.append(" .sampleRatio(").append(ratio).append(")\n"); + } + if (!otel.getBoolOrDefault("protobuf", true)) { + sb.append(" .protobuf(false)\n"); + } + if (otel.getBoolOrDefault("requireAnalyticsConsent", false)) { + sb.append(" .requireAnalyticsConsent(true)\n"); + } + for (String host : strings(otel.get("propagateTo"))) { + sb.append(" .propagateTo(").append(quote(host.trim())).append(")\n"); + } + sb.append(" );\n"); + sb.append(" }\n"); + sb.append("}\n"); + return sb.toString(); + } + + private static double ratio(Object value) { + return value instanceof Number ? ((Number) value).doubleValue() : 1.0; + } + + private static List strings(Object value) { + List out = new ArrayList(); + if (value instanceof List) { + for (Object item : (List) value) { + if (item instanceof String) { + out.add((String) item); + } + } + } else if (value instanceof String) { + out.add((String) value); + } + return out; + } + + /// RFC 9110 `token`: what a header name may be made of. + static boolean isToken(String name) { + if (name.length() == 0) { + return false; + } + for (int i = 0; i < name.length(); i++) { + char c = name.charAt(i); + boolean ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') + || "!#$%&'*+-.^_`|~".indexOf(c) >= 0; + if (!ok) { + return false; + } + } + return true; + } + + private static boolean isBlank(char c) { + return c == ' ' || c == '\t'; + } + + /// Any control character except horizontal tab, which a field value may carry. + static boolean hasControl(String value) { + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if ((c < 0x20 && c != '\t') || c == 0x7f) { + return true; + } + } + return false; + } + + /// An http or https URL WITH A HOST. The scheme alone is not a URL: "https://" + /// normalized to "https:/v1/traces", and every export -- fail-silent by design -- + /// went nowhere while the build reported the setting as checked. + static boolean isHttpUrl(String url) { + // No fragment. HTTP never sends one, so a credential kept there + // ("#api-key=...") never reached the collector: telemetry installed and + // every export was refused, silently. + if (url.indexOf('#') >= 0) { + return false; + } + // The WHOLE URL first: no space, control or DEL anywhere. Only the + // authority was checked, so a space in the path passed and every export + // then failed at transport, silently. + for (int i = 0; i < url.length(); i++) { + char c = url.charAt(i); + if (c <= 0x20 || c == 0x7f) { + return false; + } + } + int start; + if (url.regionMatches(true, 0, "http://", 0, 7)) { + start = 7; + } else if (url.regionMatches(true, 0, "https://", 0, 8)) { + start = 8; + } else { + return false; + } + int end = url.length(); + for (int i = start; i < url.length(); i++) { + char c = url.charAt(i); + if (c == '/' || c == '?' || c == '#') { + end = i; + break; + } + } + String authority = url.substring(start, end); + int at = authority.lastIndexOf('@'); + if (at >= 0 && !validUserinfo(authority.substring(0, at))) { + return false; + } + String hostPort = at >= 0 ? authority.substring(at + 1) : authority; + String host = hostPort; + if (hostPort.startsWith("[")) { + int close = hostPort.indexOf(']'); + if (close <= 1) { + return false; + } + host = hostPort.substring(1, close); + String rest = hostPort.substring(close + 1); + if (rest.length() > 0 && !validPort(rest)) { + return false; + } + } else { + int colon = hostPort.lastIndexOf(':'); + if (colon >= 0) { + if (!validPort(hostPort.substring(colon))) { + return false; + } + host = hostPort.substring(0, colon); + } + } + if (host.length() == 0) { + return false; + } + // The same host rule TelemetryConfig applies at run time, and it has to be + // the same: a URL this accepted and that refused would pass the build and + // then throw from the generated bootstrap, before Display.init, on every + // platform that does not guard it -- a crash at start-up for a mistake the + // build is there to catch. A DNS name or IPv4 address, or a bracketed IPv6 + // literal. + if (hostPort.startsWith("[")) { + return isIpv6(host); + } + for (int i = 0; i < host.length(); i++) { + char c = host.charAt(i); + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') + || "-._~".indexOf(c) >= 0)) { + return false; + } + } + return true; + } + + /// An IPv6 address by its STRUCTURE (RFC 4291 2.2), not its characters: groups + /// of one to four hex digits, at most one "::", eight groups without it and at + /// most seven with it, and optionally a dotted IPv4 tail counting as two. A + /// character check let "[:::]" through, and the transport refused every export. + static boolean isIpv6(String s) { + int n = s.length(); + if (n == 0) { + return false; + } + int groups = 0; + boolean compressed = false; + int i = 0; + if (s.startsWith("::")) { + compressed = true; + i = 2; + if (i == n) { + return true; + } + } else if (s.charAt(0) == ':') { + return false; + } + while (i < n) { + int j = i; + while (j < n && s.charAt(j) != ':') { + j++; + } + String part = s.substring(i, j); + if (part.indexOf('.') >= 0) { + if (j != n || !isIpv4(part)) { + return false; + } + groups += 2; + } else { + if (part.length() < 1 || part.length() > 4) { + return false; + } + for (int k = 0; k < part.length(); k++) { + char c = part.charAt(k); + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) { + return false; + } + } + groups++; + } + if (j == n) { + break; + } + if (j + 1 < n && s.charAt(j + 1) == ':') { + if (compressed) { + return false; + } + compressed = true; + i = j + 2; + } else { + i = j + 1; + if (i == n) { + return false; + } + } + } + return compressed ? groups <= 7 : groups == 8; + } + + /// Four decimal parts, each 0 to 255. + static boolean isIpv4(String s) { + int parts = 0; + int i = 0; + while (i <= s.length()) { + int j = s.indexOf('.', i); + if (j < 0) { + j = s.length(); + } + String part = s.substring(i, j); + if (part.length() < 1 || part.length() > 3) { + return false; + } + int value = 0; + for (int k = 0; k < part.length(); k++) { + char c = part.charAt(k); + if (c < '0' || c > '9') { + return false; + } + value = value * 10 + (c - '0'); + } + if (value > 255) { + return false; + } + parts++; + i = j + 1; + } + return parts == 4; + } + + /// RFC 3986 userinfo: unreserved characters, sub-delims, ':' and complete + /// percent escapes. Skipped over, a space, a control or a stray '%' in it passed + /// validation and failed only at transport, where the export fails silently. + static boolean validUserinfo(String userinfo) { + for (int i = 0; i < userinfo.length(); i++) { + char c = userinfo.charAt(i); + if (c == '%') { + if (i + 2 >= userinfo.length() || !isHex(userinfo.charAt(i + 1)) + || !isHex(userinfo.charAt(i + 2))) { + return false; + } + i += 2; + continue; + } + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') + || "-._~!$&'()*+,;=:".indexOf(c) >= 0)) { + return false; + } + } + return true; + } + + private static boolean isHex(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + } + + /// ":" then a TCP port, 1 through 65535. Five digits alone let 99999 through, + /// and the generated exporter, which fails silently, then lost every span. + private static boolean validPort(String colonPort) { + if (!colonPort.startsWith(":") || colonPort.length() < 2 || colonPort.length() > 6) { + return false; + } + int port = 0; + for (int i = 1; i < colonPort.length(); i++) { + char c = colonPort.charAt(i); + if (c < '0' || c > '9') { + return false; + } + port = port * 10 + (c - '0'); + } + return port >= 1 && port <= 65535; + } + + private static String quote(String value) { + StringBuilder sb = new StringBuilder("\""); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (c == '"' || c == '\\') { + sb.append('\\').append(c); + } else if (c < 0x20 || c > 0x7e) { + sb.append(String.format("\\u%04x", (int) c)); + } else { + sb.append(c); + } + } + return sb.append('"').toString(); + } +} diff --git a/maven/codenameone-maven-plugin/src/main/resources/META-INF/services/com.codename1.maven.annotations.AnnotationProcessor b/maven/codenameone-maven-plugin/src/main/resources/META-INF/services/com.codename1.maven.annotations.AnnotationProcessor index b0bf6975492..f0cdae67a7d 100644 --- a/maven/codenameone-maven-plugin/src/main/resources/META-INF/services/com.codename1.maven.annotations.AnnotationProcessor +++ b/maven/codenameone-maven-plugin/src/main/resources/META-INF/services/com.codename1.maven.annotations.AnnotationProcessor @@ -10,3 +10,4 @@ com.codename1.maven.processors.GraphQLClientAnnotationProcessor com.codename1.maven.processors.AppIntentAnnotationProcessor com.codename1.maven.processors.BuildHintAnnotationProcessor com.codename1.maven.processors.RestControllerAnnotationProcessor +com.codename1.maven.processors.TelemetryAnnotationProcessor diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/TelemetryBootstrapInstallTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/TelemetryBootstrapInstallTest.java new file mode 100644 index 00000000000..ede0d0348d4 --- /dev/null +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/TelemetryBootstrapInstallTest.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.builders; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.FileOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// The app stub installs the telemetry bootstrap exactly when the project has one. +/// Nothing else instantiates it on iOS or Android, so a missing line here is an +/// annotation that compiles, generates, and does nothing on the device. +class TelemetryBootstrapInstallTest { + + @Test + void theStubInstallsTheBootstrapWhenTheProjectHasOne(@TempDir File dir) throws Exception { + File zip = new File(dir, "app.jar"); + ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zip)); + try { + out.putNextEntry(new ZipEntry("cn1app/TelemetryBootstrap.class")); + out.write(new byte[] {(byte) 0xca, (byte) 0xfe}); + out.closeEntry(); + } finally { + out.close(); + } + String install = Executor.annotationFrameworksInstallSource(zip, " "); + assertTrue(install.contains("new cn1app.TelemetryBootstrap();"), install); + } + + @Test + void anUnpackedClassesDirectoryIsProbedLikeTheJar(@TempDir File dir) throws Exception { + // The JavaScript and native desktop builders hold the directory they + // unpacked the app into, not the jar. Asking only a jar made every + // annotation framework inert on those platforms. + touch(new File(dir, "cn1app/TelemetryBootstrap.class")); + touch(new File(dir, "cn1app/MapperBootstrap.class")); + touch(new File(dir, "com/codename1/router/generated/Routes.class")); + String install = Executor.annotationFrameworksInstallSource(dir, " "); + assertTrue(install.contains("new cn1app.TelemetryBootstrap();"), install); + assertTrue(install.contains("new cn1app.MapperBootstrap();"), install); + assertTrue(Executor.routeDispatcherInstallSource(dir, " ") + .contains("new com.codename1.router.generated.Routes();")); + } + + private static void touch(File f) throws Exception { + assertTrue(f.getParentFile().isDirectory() || f.getParentFile().mkdirs()); + new FileOutputStream(f).close(); + } + + @Test + void anAppWithoutItGetsNoInstallLine(@TempDir File dir) throws Exception { + File zip = new File(dir, "app.jar"); + ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zip)); + try { + out.putNextEntry(new ZipEntry("com/example/App.class")); + out.write(new byte[] {(byte) 0xca, (byte) 0xfe}); + out.closeEntry(); + } finally { + out.close(); + } + assertFalse(Executor.annotationFrameworksInstallSource(zip, " ") + .contains("TelemetryBootstrap")); + } +} diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java index 9dc95b63958..20770b322a3 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java @@ -224,6 +224,150 @@ public void theEntryPointDelegatesTheWholeLifecycleToTheBuilder() throws Excepti bootstrap.indexOf("HttpServer.start") < 0); } + @Test + public void theTracerIsReferencedOnlyWhenAskedFor() throws Exception { + // The whole zero-cost argument: the translator keeps what something + // reaches, and this line is the only thing that reaches the tracer. + RestControllerAnnotationProcessor untraced = new RestControllerAnnotationProcessor(); + String plain = untraced.generateBootstrap("com.example"); + assertTrue("an untraced entry point names the tracer:\n" + plain, + plain.indexOf("otel") < 0 && plain.indexOf(".tracing(") < 0); + + RestControllerAnnotationProcessor traced = new RestControllerAnnotationProcessor(); + traced.telemetry = true; + traced.telemetryServiceName = "notes"; + String bootstrap = traced.generateBootstrap("com.example"); + assertTrue("a traced entry point does not install the tracer:\n" + bootstrap, + bootstrap.indexOf(".tracing(new com.codename1.backend.otel.OtlpTracer(\"notes\"))") + >= 0); + } + + private static final String TRACED_CONTROLLER_SOURCE = CONTROLLER_SOURCE.replace( + "@RestController\n", "@RestController\n@OpenTelemetry(serviceName = \"notes\")\n"); + + @Test + public void openTelemetryOnAControllerTracesTheBuild() throws Exception { + File classes = compile(TRACED_CONTROLLER_SOURCE); + File root = sourceRootWith("Notes.java", TRACED_CONTROLLER_SOURCE); + ProcessorContext ctx = run(classes, Collections.singletonList(root.getAbsolutePath()), + tmp.newFolder()); + assertFalse(String.valueOf(ctx.getErrors()), ctx.hasErrors()); + // The router names its routes for the span: asserted on the compiled + // class, which is what ships, rather than on a source string. + assertTrue("the traced router does not name its routes", + references(new File(classes, "com/example/NotesRouter.class"), + "com/codename1/backend/Tracing")); + assertTrue("the route TEMPLATE is what names the span", + references(new File(classes, "com/example/NotesRouter.class"), + "/api/notes/{id}")); + assertTrue("the entry point does not install the tracer", + references(new File(classes, "com/example/BackendApplication.class"), + "com/codename1/backend/otel/OtlpTracer")); + } + + @Test + public void aBlankServiceNameIsNoServiceName() throws Exception { + // " " is not a name. Kept raw it reached the entry point, and the server + // reported an empty service.name instead of unknown_service. + String source = CONTROLLER_SOURCE.replace("@RestController\n", + "@RestController\n@OpenTelemetry(serviceName = \" \")\n"); + File classes = compile(source); + File root = sourceRootWith("Notes.java", source); + Map index = ClassScanner.scan(classes); + ProcessorContext ctx = new ProcessorContext(classes, tmp.newFolder(), index, + new SystemStreamLog(), tmp.newFolder(), new Properties(), null, + Collections.singletonList(root.getAbsolutePath()), "UTF-8", + Collections.emptyList()); + RestControllerAnnotationProcessor proc = new RestControllerAnnotationProcessor(); + proc.resolveTelemetry(ctx); + assertFalse(String.valueOf(ctx.getErrors()), ctx.hasErrors()); + assertTrue("the annotation still turns tracing on", proc.telemetry); + assertTrue("a blank name was kept: '" + proc.telemetryServiceName + "'", + proc.telemetryServiceName == null || proc.telemetryServiceName.length() == 0); + assertTrue(proc.generateBootstrap("com.example") + .indexOf(".tracing(new com.codename1.backend.otel.OtlpTracer(null))") >= 0); + } + + @Test + public void anUntracedBuildNamesRoutesButNeverLinksTheTracer() throws Exception { + // The same controller without the annotation. The OTLP tracer is not + // linked -- that is what the build switch buys -- but the router still + // names its routes, for a tracer the program installs itself. + File classes = compile(CONTROLLER_SOURCE); + File root = sourceRootWith("Notes.java", CONTROLLER_SOURCE); + ProcessorContext ctx = run(classes, Collections.singletonList(root.getAbsolutePath()), + tmp.newFolder()); + assertFalse(String.valueOf(ctx.getErrors()), ctx.hasErrors()); + assertTrue("a manually installed tracer would see every route merged under its method", + references(new File(classes, "com/example/NotesRouter.class"), + "com/codename1/backend/Tracing")); + assertFalse(references(new File(classes, "com/example/NotesRouter.class"), + "com/codename1/backend/otel/OtlpTracer")); + assertFalse(references(new File(classes, "com/example/BackendApplication.class"), + "com/codename1/backend/otel/OtlpTracer")); + } + + @Test + public void thePropertyTracesTheBuildWithoutTheAnnotation() throws Exception { + File classes = compile(CONTROLLER_SOURCE); + File root = sourceRootWith("Notes.java", CONTROLLER_SOURCE); + File project = tmp.newFolder(); + writeUtf8(new File(project, "application.properties"), "cn1.otel.enabled=true\n"); + ProcessorContext ctx = run(classes, Collections.singletonList(root.getAbsolutePath()), + project); + assertFalse(String.valueOf(ctx.getErrors()), ctx.hasErrors()); + assertTrue(references(new File(classes, "com/example/BackendApplication.class"), + "com/codename1/backend/otel/OtlpTracer")); + // A reference the build cannot resolve is not a yes. + File unresolved = compile(CONTROLLER_SOURCE); + File other = tmp.newFolder(); + writeUtf8(new File(other, "application.properties"), + "cn1.otel.enabled=${TRACING}\n"); + run(unresolved, Collections.singletonList(root.getAbsolutePath()), other); + assertFalse(references(new File(unresolved, "com/example/BackendApplication.class"), + "com/codename1/backend/otel/OtlpTracer")); + } + + @Test + public void thePropertyInTheStandardResourcesPlaceTracesTheBuild() throws Exception { + // src/main/resources/application.properties, which process-resources has + // copied into the classes directory before this goal runs. Looking only + // beside the module missed it, so the documented layout never traced. + File classes = compile(CONTROLLER_SOURCE); + File root = sourceRootWith("Notes.java", CONTROLLER_SOURCE); + writeUtf8(new File(classes, "application.properties"), "cn1.otel.enabled=true\n"); + ProcessorContext ctx = run(classes, Collections.singletonList(root.getAbsolutePath()), + tmp.newFolder()); + assertFalse(String.valueOf(ctx.getErrors()), ctx.hasErrors()); + assertTrue("cn1.otel.enabled in the resources copy did not trace the build", + references(new File(classes, "com/example/BackendApplication.class"), + "com/codename1/backend/otel/OtlpTracer")); + + // And straight from the source tree, for a build that skipped resources. + File untouched = compile(CONTROLLER_SOURCE); + File project = tmp.newFolder(); + writeUtf8(new File(project, "src/main/resources/application.properties"), + "cn1.otel.enabled=true\n"); + run(untouched, Collections.singletonList(root.getAbsolutePath()), project); + assertTrue(references(new File(untouched, "com/example/BackendApplication.class"), + "com/codename1/backend/otel/OtlpTracer")); + } + + private File sourceRootWith(String file, String source) throws Exception { + File root = tmp.newFolder(); + File pkg = new File(root, "com/example"); + assertTrue(pkg.mkdirs()); + writeUtf8(new File(pkg, file), source); + return root; + } + + /** Whether a class file's constant pool holds this text. */ + private static boolean references(File classFile, String text) throws Exception { + assertTrue("not generated: " + classFile, classFile.isFile()); + byte[] bytes = java.nio.file.Files.readAllBytes(classFile.toPath()); + return new String(bytes, "ISO-8859-1").indexOf(text) >= 0; + } + @Test public void ignoresAControllerWhoseSourceIsGone() throws Exception { // Maven does not clean target/classes on its own, so the .class of a @@ -314,6 +458,7 @@ public void removesTheMainMarkerWhenNoControllerRemains() throws Exception { } private static void writeUtf8(File f, String text) throws Exception { + f.getParentFile().mkdirs(); java.io.OutputStream out = new java.io.FileOutputStream(f); try { out.write(text.getBytes("UTF-8")); @@ -1992,6 +2137,12 @@ private ProcessorContext run(File classes) throws Exception { /// front of a class whose `.java` is NOT among them -- a `target/classes` /// left over from before the source was deleted or renamed. private ProcessorContext run(File classes, List compileSourceRoots) throws Exception { + return run(classes, compileSourceRoots, tmp.newFolder()); + } + + /// With the module directory given, where `application.properties` is read. + private ProcessorContext run(File classes, List compileSourceRoots, File projectDir) + throws Exception { Map index = ClassScanner.scan(classes); RestControllerAnnotationProcessor proc = new RestControllerAnnotationProcessor(); List cp = new java.util.ArrayList(); @@ -1999,7 +2150,7 @@ private ProcessorContext run(File classes, List compileSourceRoots) thro cp.add(f.getAbsolutePath()); } ProcessorContext ctx = new ProcessorContext(classes, tmp.newFolder(), index, - new SystemStreamLog(), tmp.newFolder(), new Properties(), null, + new SystemStreamLog(), projectDir, new Properties(), null, compileSourceRoots, "UTF-8", cp); proc.start(ctx); for (AnnotatedClass cls : index.values()) { diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/TelemetryAnnotationProcessorTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/TelemetryAnnotationProcessorTest.java new file mode 100644 index 00000000000..f20738e6bf8 --- /dev/null +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/TelemetryAnnotationProcessorTest.java @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven.processors; + +import com.codename1.maven.annotations.AnnotatedClass; +import com.codename1.maven.annotations.ClassScanner; +import com.codename1.maven.annotations.JavaSourceCompiler; +import com.codename1.maven.annotations.ProcessorContext; + +import org.apache.maven.plugin.logging.SystemStreamLog; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.net.URL; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/// `@OpenTelemetry` on the app: the bootstrap it generates, compiled against the +/// real core, and every configuration mistake the build can see refused there. +public class TelemetryAnnotationProcessorTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + @Test + public void aRelayConfigurationBecomesTheBootstrap() throws Exception { + File classes = compile("@OpenTelemetry(relay = \"https://api.example.com\", " + + "serviceName = \"shop-app\", relayToken = \"t0k\", sampleRatio = 0.25, " + + "propagateTo = {\"cdn.example.com\"})"); + ProcessorContext ctx = run(classes); + assertFalse(String.valueOf(ctx.getErrors()), ctx.hasErrors()); + assertTrue("the bootstrap was not compiled", + new File(classes, "cn1app/TelemetryBootstrap.class").isFile()); + String source = bootstrapSource(classes); + assertContains(source, "com.codename1.telemetry.Telemetry.install("); + assertContains(source, ".relay(\"https://api.example.com\")"); + assertContains(source, ".serviceName(\"shop-app\")"); + assertContains(source, ".relayToken(\"t0k\")"); + assertContains(source, ".sampleRatio(0.25)"); + assertContains(source, ".propagateTo(\"cdn.example.com\")"); + assertFalse("consent is opt-in, not the default", source.contains("requireAnalyticsConsent")); + } + + @Test + public void theConsentFlagReachesTheBootstrap() throws Exception { + File classes = compile("@OpenTelemetry(relay = \"https://api.example.com\", " + + "requireAnalyticsConsent = true)"); + ProcessorContext ctx = run(classes); + assertFalse(String.valueOf(ctx.getErrors()), ctx.hasErrors()); + assertContains(bootstrapSource(classes), ".requireAnalyticsConsent(true)"); + } + + @Test + public void aDirectConfigurationCarriesItsHeaders() throws Exception { + File classes = compile("@OpenTelemetry(endpoint = \"https://collector.example.com:4318\", " + + "headers = \"Authorization: Api-Token dt0c01.x\", protobuf = false)"); + ProcessorContext ctx = run(classes); + assertFalse(String.valueOf(ctx.getErrors()), ctx.hasErrors()); + String source = bootstrapSource(classes); + assertContains(source, ".direct(\"https://collector.example.com:4318\")"); + assertContains(source, ".header(\"Authorization\", \"Api-Token dt0c01.x\")"); + assertContains(source, ".protobuf(false)"); + } + + @Test + public void anAppWithoutTheAnnotationGetsNoBootstrap() throws Exception { + File classes = compile(""); + ProcessorContext ctx = run(classes); + assertFalse(ctx.hasErrors()); + assertFalse("an app that never asked carries a telemetry bootstrap", + new File(classes, "cn1app/TelemetryBootstrap.class").exists()); + } + + @Test + public void removingTheAnnotationRemovesTheBootstrapAnEarlierBuildLeft() throws Exception { + // Maven keeps target/classes between builds, and the builders install + // whatever bootstrap class is there. + File classes = compile(""); + Map earlier = new LinkedHashMap(); + earlier.put("cn1app.TelemetryBootstrap", + "package cn1app;\npublic final class TelemetryBootstrap {\n}\n"); + JavaSourceCompiler.compile(earlier, classes, Arrays.asList(testClassesDir())); + File stale = new File(classes, "cn1app/TelemetryBootstrap.class"); + assertTrue("the fixture did not compile", stale.isFile()); + ProcessorContext ctx = run(classes); + assertFalse(ctx.hasErrors()); + assertFalse("a bootstrap for telemetry nobody asks for any more was left to ship", + stale.exists()); + } + + @Test + public void nowhereToSendIsRefused() throws Exception { + assertRefused("@OpenTelemetry(serviceName = \"x\")"); + } + + @Test + public void bothARelayAndACollectorIsRefused() throws Exception { + assertRefused("@OpenTelemetry(relay = \"https://a\", endpoint = \"https://b\")"); + } + + @Test + public void aCredentialOnARelayIsRefused() throws Exception { + // The relay exists so the credential stays on the server; a header here + // would ship it in the app while looking as though it did not. + assertRefused("@OpenTelemetry(relay = \"https://a\", headers = \"Authorization: x\")"); + } + + @Test + public void aMalformedHeaderIsRefused() throws Exception { + assertRefused("@OpenTelemetry(endpoint = \"https://a\", headers = \"Authorization\")"); + } + + @Test + public void aHeaderThatIsNotValidHttpIsRefused() throws Exception { + assertRefused("@OpenTelemetry(endpoint = \"https://a\", headers = \"Bad Name: v\")"); + assertRefused("@OpenTelemetry(endpoint = \"https://a\", " + + "headers = \"X-Token: a\\r\\nInjected: b\")"); + } + + @Test + public void aRatioOutOfRangeIsRefused() throws Exception { + assertRefused("@OpenTelemetry(endpoint = \"https://a\", sampleRatio = 2)"); + } + + @Test + public void aUrlThatIsNotHttpIsRefused() throws Exception { + assertRefused("@OpenTelemetry(endpoint = \"collector:4318\")"); + assertRefused("@OpenTelemetry(endpoint = \"https://\")"); + assertRefused("@OpenTelemetry(relay = \"https:///path\")"); + assertRefused("@OpenTelemetry(endpoint = \"https://host:port/\")"); + assertTrue(TelemetryAnnotationProcessor.isHttpUrl("https://collector.example:4318/otlp")); + assertTrue(TelemetryAnnotationProcessor.isHttpUrl("http://[::1]:4318")); + // A port outside TCP's range is five digits too. + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://collector.example:99999")); + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://collector.example:0")); + assertTrue(TelemetryAnnotationProcessor.isHttpUrl("https://collector.example:65535")); + // The runtime's host rule, so nothing the build accepts throws at start-up. + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://collector!x.example")); + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://[not-ipv6]:4318")); + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://[1234]")); + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://bad value@collector.example")); + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://u%zz@collector.example")); + assertTrue(TelemetryAnnotationProcessor.isHttpUrl("https://user:p%40ss@collector.example")); + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://collector.example/bad path")); + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://[:::]:4318")); + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://[1::2::3]:4318")); + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://[1:2:3:4:5:6:7:8:9]:4318")); + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://[::ffff:300.0.0.1]:4318")); + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://[1:2:3:4:5:6:7]:4318")); + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://[12345::1]:4318")); + assertTrue(TelemetryAnnotationProcessor.isHttpUrl("https://[::1]:4318")); + assertTrue(TelemetryAnnotationProcessor.isHttpUrl("https://[::ffff:10.0.0.7]:4318")); + assertTrue(TelemetryAnnotationProcessor.isHttpUrl("https://[2001:db8::1]:4318")); + assertTrue(TelemetryAnnotationProcessor.isHttpUrl("https://[1:2:3:4:5:6:7:8]:4318")); + assertTrue(TelemetryAnnotationProcessor.isHttpUrl("https://[::]:4318")); + } + + @Test + public void aMalformedHeaderIsRefusedWithoutQuotingIt() throws Exception { + File classes = compile("@OpenTelemetry(endpoint = \"https://c.example\", " + + "headers = \"Authorization Bearer s3cret\")"); + ProcessorContext ctx = run(classes); + assertTrue(ctx.hasErrors()); + assertFalse("the refusal quoted a credential: " + ctx.getErrors(), + String.valueOf(ctx.getErrors()).contains("s3cret")); + } + + @Test + public void anEndpointWithAFragmentIsRefused() throws Exception { + assertFalse(TelemetryAnnotationProcessor.isHttpUrl("https://c.example/v1#api-key=s3cret")); + } + + @Test + public void aHeaderTheExporterOwnsIsRefused() throws Exception { + assertRefused("@OpenTelemetry(endpoint = \"https://c.example\", headers = \"Content-Type: application/json\")"); + } + + @Test + public void aRelayTokenNoHeaderMayCarryIsRefused() throws Exception { + // TelemetryConfig would throw on it inside the bootstrap, before + // Display.init; the build says so instead. + assertRefused("@OpenTelemetry(relay = \"https://api.example\", relayToken = \"bad\\nvalue\")"); + assertRefused("@OpenTelemetry(relay = \"https://api.example\", relayToken = \"trailing \")"); + assertRefused("@OpenTelemetry(relay = \"https://api.example\", relayToken = \" leading\")"); + } + + // ------------------------------------------------------------------ + + private void assertRefused(String annotation) throws Exception { + File classes = compile(annotation); + ProcessorContext ctx = run(classes); + assertTrue("accepted " + annotation, ctx.hasErrors()); + assertFalse(new File(classes, "cn1app/TelemetryBootstrap.class").exists()); + } + + private static void assertContains(String source, String expected) { + assertTrue("expected " + expected + " in:\n" + source, source.contains(expected)); + } + + private File compile(String annotation) throws Exception { + File classes = tmp.newFolder(); + Map sources = new LinkedHashMap(); + sources.put("com.example.ShopApp", + "package com.example;\n" + + "import com.codename1.annotations.OpenTelemetry;\n" + + annotation + "\n" + + "public class ShopApp {\n" + + "}\n"); + JavaSourceCompiler.compile(sources, classes, Arrays.asList(testClassesDir())); + return classes; + } + + private ProcessorContext run(File classes) throws Exception { + Map index = ClassScanner.scan(classes); + TelemetryAnnotationProcessor proc = new TelemetryAnnotationProcessor(); + ProcessorContext ctx = new ProcessorContext(classes, tmp.newFolder(), index, + new SystemStreamLog()); + proc.start(ctx); + for (AnnotatedClass cls : index.values()) { + if (!cls.getClassAnnotations().isEmpty()) { + proc.processClass(cls, ctx); + } + } + proc.finish(ctx); + return ctx; + } + + private static String bootstrapSource(File classes) throws Exception { + Map index = ClassScanner.scan(classes); + for (AnnotatedClass cls : index.values()) { + if (cls.getClassAnnotation(TelemetryAnnotationProcessor.OPEN_TELEMETRY_DESC) != null) { + return TelemetryAnnotationProcessor.generateBootstrapSource( + cls.getClassAnnotation(TelemetryAnnotationProcessor.OPEN_TELEMETRY_DESC)); + } + } + throw new AssertionError("no @OpenTelemetry class in " + index.keySet()); + } + + private static File testClassesDir() throws Exception { + URL url = TelemetryAnnotationProcessorTest.class.getProtectionDomain() + .getCodeSource().getLocation(); + return new File(url.toURI()); + } +} diff --git a/maven/core-unittests/pom.xml b/maven/core-unittests/pom.xml index f4a9bfc2347..aadaa09d05b 100644 --- a/maven/core-unittests/pom.xml +++ b/maven/core-unittests/pom.xml @@ -205,5 +205,21 @@ ${project.version} test + + + io.opentelemetry.proto + opentelemetry-proto + 1.3.2-alpha + test + + + com.google.protobuf + protobuf-java + 3.25.5 + test + diff --git a/maven/core-unittests/src/test/java/com/codename1/io/NetworkTracerQueueTest.java b/maven/core-unittests/src/test/java/com/codename1/io/NetworkTracerQueueTest.java new file mode 100644 index 00000000000..0c6452795c2 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/io/NetworkTracerQueueTest.java @@ -0,0 +1,375 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.io; + +import com.codename1.junit.UITestBase; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// What `NetworkManager.addToQueue` records for the tracer, checked on a private +/// manager whose queue nothing consumes. Driving the shared manager through a busy +/// network thread instead was not deterministic: other test classes leave extra +/// network threads consuming the shared queue, so "still pending" could not be +/// arranged reliably. +class NetworkTracerQueueTest extends UITestBase { + + @AfterEach + void removeTracer() { + NetworkManager.setNetworkTracer(null); + } + + @Test + void aRejectedDuplicateEnqueueKeepsTheQueuedRequestsParent() throws Exception { + NetworkManager manager = idleManager(); + final int[] calls = new int[1]; + NetworkTracer tracer = new NetworkTracer() { + @Override + public Object requestQueued(ConnectionRequest request) { + calls[0]++; + return calls[0] == 1 ? "first action" : "second action"; + } + + @Override + public Object beforeRequest(ConnectionRequest request, Object parent) { + return null; + } + + @Override + public void afterRequest(ConnectionRequest request, Object attempt, int status, + Throwable error) { + } + }; + NetworkManager.setNetworkTracer(tracer); + ConnectionRequest request = new ConnectionRequest(); + request.setUrl("http://queue.test/"); + request.setDuplicateSupported(false); + + manager.addToQueue(request, false); + assertEquals("first action", request.tracerParent); + assertSame(tracer, request.tracerParentOwner); + + // The same request again while it is pending: rejected as a duplicate, + // and it must keep the context of the enqueue that was accepted. + manager.addToQueue(request, false); + assertEquals(2, calls[0], "the second enqueue did ask the tracer"); + assertEquals("first action", request.tracerParent, + "a rejected duplicate re-parented the queued request"); + } + + @Test + void aRetryKeepsItsContextWithoutAskingAgain() throws Exception { + NetworkManager manager = idleManager(); + final int[] calls = new int[1]; + NetworkManager.setNetworkTracer(new NetworkTracer() { + @Override + public Object requestQueued(ConnectionRequest request) { + calls[0]++; + return "action"; + } + + @Override + public Object beforeRequest(ConnectionRequest request, Object parent) { + return null; + } + + @Override + public void afterRequest(ConnectionRequest request, Object attempt, int status, + Throwable error) { + } + }); + ConnectionRequest request = new ConnectionRequest(); + request.setUrl("http://queue.test/retry"); + manager.addToQueue(request, false); + manager.addToQueue(request, true); + assertEquals(1, calls[0]); + assertEquals("action", request.tracerParent); + } + + @Test + void aRetryEndsTheCurrentAttemptBeforeTheRequestIsQueuedAgain() throws Exception { + // With several network threads another worker can run the re-queued + // request before this one reaches its finally; the attempt must already + // be over by then, or the next attempt overwrites it and the span is lost. + NetworkManager manager = idleManager(); + final Object[] ended = new Object[1]; + NetworkTracer tracer = new NetworkTracer() { + @Override + public Object requestQueued(ConnectionRequest request) { + return null; + } + + @Override + public Object beforeRequest(ConnectionRequest request, Object parent) { + return null; + } + + @Override + public void afterRequest(ConnectionRequest request, Object attempt, int status, + Throwable error) { + ended[0] = attempt; + } + }; + final ConnectionRequest request = new ConnectionRequest(); + request.setUrl("http://queue.test/redirected"); + request.tracerAttempt = "first attempt"; + request.tracerOwner = tracer; + request.tracerThread = Thread.currentThread(); + + // A thread that did not start the attempt -- the worker running the NEXT + // one reaching this attempt's finally late -- must not end it. + Thread other = new Thread(new Runnable() { + @Override + public void run() { + NetworkManager.endTracerAttempt(request, null); + } + }); + other.start(); + other.join(); + assertEquals(null, ended[0], "another thread ended an attempt it did not run"); + + manager.addToQueue(request, true); + assertEquals("first attempt", ended[0], "the retry left the attempt open"); + assertEquals(null, request.tracerAttempt); + assertEquals(null, request.tracerOwner); + } + + @Test + void aFinishedRequestLetsGoOfItsTracerState() throws Exception { + // The parent and the last attempt are the tracer's objects -- through them, + // a whole telemetry installation. A request kept for reuse must not hold + // them once its last attempt is over. + com.codename1.testing.TestCodenameOneImplementation.getInstance() + .addNetworkMockResponse("http://queue.test/done", 200, "OK", new byte[0]); + NetworkManager.setNetworkTracer(new NetworkTracer() { + @Override + public Object requestQueued(ConnectionRequest request) { + return "the action"; + } + + @Override + public Object beforeRequest(ConnectionRequest request, Object parent) { + return "the attempt"; + } + + @Override + public void afterRequest(ConnectionRequest request, Object attempt, int status, + Throwable error) { + } + }); + ConnectionRequest request = new ConnectionRequest() { + @Override + protected void readResponse(java.io.InputStream input) { + } + }; + request.setUrl("http://queue.test/done"); + request.setPost(false); + NetworkManager.getInstance().addToQueueAndWait(request); + assertEquals(null, request.tracerParent); + assertEquals(null, request.tracerParentOwner); + assertEquals(null, request.tracerLastAttempt); + assertEquals(null, request.tracerLastOwner); + + // Owner-only state: nothing captured, no attempt made -- the owner alone + // pins the installation and must go too. + NetworkManager.setNetworkTracer(new NetworkTracer() { + public Object requestQueued(ConnectionRequest r) { return null; } + public Object beforeRequest(ConnectionRequest r, Object p) { return null; } + public void afterRequest(ConnectionRequest r, Object at, int s, Throwable e) { } + }); + ConnectionRequest untraced = new ConnectionRequest() { + @Override + protected void readResponse(java.io.InputStream input) { + } + }; + untraced.setUrl("http://queue.test/done"); + untraced.setPost(false); + NetworkManager.getInstance().addToQueueAndWait(untraced); + flushSerialCalls(); + assertEquals(null, untraced.tracerParentOwner, "the tracer owner outlived the request"); + } + + @Test + void aRequestKilledBeforeItStartsLetsGoOfItsTracerState() throws Exception { + // kill() on a pending request leaves it in the queue; the worker takes it + // and skips it without running it, so runCurrentRequest's cleanup never + // runs. That skip must forget the tracer state, and the worker must not + // keep the dead request as its current one. + final NetworkManager manager = idleManager(); + NetworkManager.setNetworkTracer(new NetworkTracer() { + @Override + public Object requestQueued(ConnectionRequest request) { + return "the action"; + } + + @Override + public Object beforeRequest(ConnectionRequest request, Object parent) { + return "the attempt"; + } + + @Override + public void afterRequest(ConnectionRequest request, Object attempt, int status, + Throwable error) { + } + }); + ConnectionRequest request = new ConnectionRequest(); + request.setUrl("http://queue.test/killed"); + manager.addToQueue(request, false); + assertEquals("the action", request.tracerParent); + request.kill(); + + Field threadsField = NetworkManager.class.getDeclaredField("networkThreads"); + threadsField.setAccessible(true); + final NetworkManager.NetworkThread worker = + ((NetworkManager.NetworkThread[]) threadsField.get(manager))[0]; + Field pendingField = NetworkManager.class.getDeclaredField("pending"); + pendingField.setAccessible(true); + java.util.Vector pending = (java.util.Vector) pendingField.get(manager); + Thread runner = new Thread(worker); + runner.start(); + try { + long deadline = System.currentTimeMillis() + 10000; + while (!pending.isEmpty() && System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + assertTrue(pending.isEmpty(), "the worker never took the killed request"); + } finally { + set(manager, "running", Boolean.FALSE); + Field lockField = NetworkManager.class.getDeclaredField("LOCK"); + lockField.setAccessible(true); + Object lock = lockField.get(null); + synchronized (lock) { + lock.notifyAll(); + } + runner.join(10000); + } + // The worker removes the request and handles the skip inside one hold of + // the queue lock, so an empty queue means the skip has run. + assertEquals(null, worker.getCurrentRequest(), "the worker kept the killed request"); + flushSerialCalls(); + assertEquals(null, request.tracerParent, "a killed request kept its parent"); + assertEquals(null, request.tracerParentOwner, "a killed request kept its tracer"); + } + + @Test + void everyAcceptedEnqueueAdvancesTheGeneration() throws Exception { + // A fresh reuse as well as a retry: a cleanup the previous run queued on + // the EDT compares against this, and must not clear the new run's parent. + NetworkManager manager = idleManager(); + ConnectionRequest request = new ConnectionRequest(); + request.setUrl("http://queue.test/generation"); + int before = request.tracerRequeues; + manager.addToQueue(request, false); + assertEquals(before + 1, request.tracerRequeues, "a fresh enqueue left the generation"); + manager.addToQueue(request, true); + assertEquals(before + 2, request.tracerRequeues); + } + + @Test + void aRetryChainsPastAParentAnotherTracerCaptured() throws Exception { + // Queued under tracer A, run under tracer B: B's attempts must still + // chain, not each start a new root because A's parent is in the way. + NetworkManager manager = idleManager(); + NetworkTracer a = new NetworkTracer() { + public Object requestQueued(ConnectionRequest r) { return null; } + public Object beforeRequest(ConnectionRequest r, Object p) { return null; } + public void afterRequest(ConnectionRequest r, Object at, int s, Throwable e) { } + }; + NetworkTracer b = new NetworkTracer() { + public Object requestQueued(ConnectionRequest r) { return null; } + public Object beforeRequest(ConnectionRequest r, Object p) { return null; } + public void afterRequest(ConnectionRequest r, Object at, int s, Throwable e) { } + }; + ConnectionRequest request = new ConnectionRequest(); + request.setUrl("http://queue.test/foreign"); + request.tracerParent = "A's context"; + request.tracerParentOwner = a; + request.tracerLastAttempt = "B's first attempt"; + request.tracerLastOwner = b; + manager.addToQueue(request, true); + assertEquals("B's first attempt", request.tracerParent); + assertSame(b, request.tracerParentOwner); + } + + @Test + void aRetryFromAnotherThreadChainsToTheAttemptStillInFlight() throws Exception { + // A listener retrying from the EDT can beat the network thread to the end + // of the attempt it is reacting to. That attempt has not ended -- there is + // no "last attempt" yet -- and the retry must still continue it. + NetworkManager manager = idleManager(); + NetworkTracer tracer = new NetworkTracer() { + public Object requestQueued(ConnectionRequest r) { return null; } + public Object beforeRequest(ConnectionRequest r, Object p) { return null; } + public void afterRequest(ConnectionRequest r, Object at, int s, Throwable e) { } + }; + ConnectionRequest request = new ConnectionRequest(); + request.setUrl("http://queue.test/in-flight"); + request.tracerAttempt = "the attempt still running"; + request.tracerOwner = tracer; + request.tracerThread = new Thread(); // not this thread: it cannot end it + manager.addToQueue(request, true); + assertEquals("the attempt still running", request.tracerParent, + "a retry that beat the network thread started a new trace"); + assertSame(tracer, request.tracerParentOwner); + } + + @Test + void addIfAbsentLeavesAnExplicitContentTypeAlone() { + ConnectionRequest request = new ConnectionRequest(); + request.setContentType("application/json"); + assertEquals(false, request.addRequestHeaderIfAbsent("Content-Type", "text/plain")); + assertEquals("application/json", request.getContentType()); + assertEquals("application/json", request.getRequestHeader("content-type")); + + ConnectionRequest plain = new ConnectionRequest(); + assertEquals(null, plain.getRequestHeader("Content-Type"), + "the default content type is not a header anyone added"); + assertEquals(true, plain.addRequestHeaderIfAbsent("X-Trace", "1")); + assertEquals("1", plain.getRequestHeader("x-trace")); + } + + /// A manager that queues and never runs anything: marked running with one + /// network thread that was never started. + private static NetworkManager idleManager() throws Exception { + Constructor constructor = NetworkManager.class.getDeclaredConstructor(); + constructor.setAccessible(true); + NetworkManager manager = constructor.newInstance(); + set(manager, "running", Boolean.TRUE); + set(manager, "autoDetected", Boolean.TRUE); + set(manager, "networkThreads", new NetworkManager.NetworkThread[] {manager.new NetworkThread()}); + return manager; + } + + private static void set(Object target, String name, Object value) throws Exception { + Field field = NetworkManager.class.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/junit/UITestBase.java b/maven/core-unittests/src/test/java/com/codename1/junit/UITestBase.java index 8d89ed6f0ce..d2ece9f0574 100644 --- a/maven/core-unittests/src/test/java/com/codename1/junit/UITestBase.java +++ b/maven/core-unittests/src/test/java/com/codename1/junit/UITestBase.java @@ -183,6 +183,23 @@ protected void tearDownDisplay() throws Exception { // the singleton reset above cannot reach. resetDisplayBooleanArrayField("selectionPressed"); resetDisplayIntField("dragPathLength", 0); + resetNetworkErrorListeners(); + } + + /// Drops the NetworkManager's GLOBAL error listeners. Every test shares the one + /// manager, and the framework itself registers listeners that CONSUME errors -- + /// Lifecycle.init does, so does ToastBar -- so any test that ran an app's init + /// left one behind, and from then on no IOException reached a request's own + /// handleIOException in any later test. A test that needs a listener adds it + /// itself. + private static void resetNetworkErrorListeners() { + try { + Field errors = com.codename1.io.NetworkManager.class.getDeclaredField("errorListeners"); + errors.setAccessible(true); + errors.set(com.codename1.io.NetworkManager.getInstance(), null); + } catch (Exception ignored) { + // A renamed field loses the reset, not the test run. + } } /// Takes down any native window the test left open. diff --git a/maven/core-unittests/src/test/java/com/codename1/telemetry/TelemetryTest.java b/maven/core-unittests/src/test/java/com/codename1/telemetry/TelemetryTest.java new file mode 100644 index 00000000000..65fa959eb02 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/telemetry/TelemetryTest.java @@ -0,0 +1,1293 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.telemetry; + +import com.codename1.analytics.Analytics; +import com.codename1.analytics.AnalyticsConsent; +import com.codename1.analytics.ConsentMode; +import com.codename1.io.ConnectionRequest; +import com.codename1.io.NetworkManager; +import com.codename1.junit.UITestBase; +import com.codename1.testing.TestCodenameOneImplementation; + +import com.google.protobuf.ByteString; + +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.trace.v1.Span; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/// The app's side of distributed tracing, through the real NetworkManager against +/// the mocked network: what a request carries to the server, and what reaches the +/// collector -- decoded with the specification's own generated classes. +class TelemetryTest extends UITestBase { + private static final String API = "http://api.test/pets"; + private static final String COLLECTOR = "http://collector.test/v1/traces"; + private static final String RELAY = "http://backend.test/otel/v1/traces"; + /// How long a test waits for something the network thread does. An upper + /// bound, never a delay: every wait below leaves the moment its condition + /// holds. Generous because the whole suite shares one NetworkManager, and a + /// request an earlier class left in its queue can hold this one's back -- + /// a 5s bound failed there while every test passed on its own. + private static final long WAIT_MILLIS = 30000; + + @BeforeEach + void mocks() { + TestCodenameOneImplementation impl = TestCodenameOneImplementation.getInstance(); + impl.clearNetworkMocks(); + impl.clearConnections(); + impl.addNetworkMockResponse(API, 200, "OK", "[]".getBytes(StandardCharsets.UTF_8)); + impl.addNetworkMockResponse("http://api.test/missing", 404, "Not Found", new byte[0]); + impl.addNetworkMockResponse(COLLECTOR, 200, "OK", new byte[0]); + impl.addNetworkMockResponse(RELAY, 200, "OK", "{}".getBytes(StandardCharsets.UTF_8)); + } + + @AfterEach + void uninstall() { + Telemetry.uninstall(); + TestCodenameOneImplementation.getInstance().clearNetworkMocks(); + } + + @Test + void aRequestCarriesTheTraceAndBecomesAChildSpan() throws Exception { + Telemetry.install(new TelemetryConfig().direct("http://collector.test") + .serviceName("shop-app").header("Authorization", "Api-Token t0k")); + final TelemetrySpan[] action = new TelemetrySpan[1]; + Telemetry.run("checkout", new Runnable() { + @Override + public void run() { + action[0] = Telemetry.getCurrentSpan(); + NetworkManager.getInstance().addToQueueAndWait(request(API + "?token=secret")); + } + }); + assertNull(Telemetry.getCurrentSpan(), "run() restores the previous span"); + + TestCodenameOneImplementation.TestConnection api = connection(API + "?token=secret"); + String traceparent = api.getHeaders().get("traceparent"); + assertNotNull(traceparent, "the request did not carry the trace context"); + assertTrue(traceparent.startsWith("00-" + action[0].getTraceId() + "-"), + "the request is part of the action's trace: " + traceparent); + assertTrue(traceparent.endsWith("-01")); + + Telemetry.flush(); + List spans = exported(2); + Span get = find(spans, "GET"); + Span checkout = find(spans, "checkout"); + assertEquals(action[0].getSpanId(), hex(get.getParentSpanId())); + assertEquals(hex(get.getSpanId()), traceparent.substring(36, 52), + "the header names the request's own span as the server's parent"); + assertEquals(Span.SpanKind.SPAN_KIND_CLIENT, get.getKind()); + assertEquals(Span.SpanKind.SPAN_KIND_INTERNAL, checkout.getKind()); + assertEquals(0x101, get.getFlags(), "sampled, with a parent known to be local"); + assertEquals(1, checkout.getFlags(), "a root claims nothing about a parent it lacks"); + assertEquals(API, attribute(get.getAttributesList(), "url.full"), + "the query string is never recorded"); + // One clock per trace: the request sits inside the action that caused it. + assertTrue(get.getStartTimeUnixNano() >= checkout.getStartTimeUnixNano()); + assertTrue(get.getEndTimeUnixNano() <= checkout.getEndTimeUnixNano()); + assertEquals("200", attribute(get.getAttributesList(), "http.response.status_code")); + + TestCodenameOneImplementation.TestConnection export = connection(COLLECTOR); + assertEquals("Api-Token t0k", export.getHeaders().get("Authorization")); + assertNull(export.getHeaders().get("traceparent"), + "the export's own request must not be traced"); + } + + @Test + void aFailedRequestIsAnErrorSpan() throws Exception { + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + ConnectionRequest missing = request("http://api.test/missing"); + missing.setFailSilently(true); + NetworkManager.getInstance().addToQueueAndWait(missing); + Telemetry.flush(); + Span span = find(exported(1), "GET"); + assertEquals("404", attribute(span.getAttributesList(), "http.response.status_code")); + assertEquals(io.opentelemetry.proto.trace.v1.Status.StatusCode.STATUS_CODE_ERROR, + span.getStatus().getCode()); + assertEquals(0, span.getParentSpanId().size(), "a request outside any action is a root"); + } + + @Test + void theRelayGetsJsonAndTheTokenAndNeverTheCredential() throws Exception { + Telemetry.install(new TelemetryConfig().relay("http://backend.test") + .relayToken("r3lay").header("Authorization", "never-sent")); + NetworkManager.getInstance().addToQueueAndWait(request(API)); + Telemetry.flush(); + TestCodenameOneImplementation.TestConnection relay = awaitConnection(RELAY); + assertEquals("r3lay", relay.getHeaders().get("X-CN1-Telemetry-Token")); + assertNull(relay.getHeaders().get("Authorization"), + "a relay export must not carry a collector credential"); + String json = new String(relay.getOutputData(), StandardCharsets.UTF_8); + assertTrue(json.startsWith("{\"resourceSpans\":[{\"resource\":{\"attributes\":"), json); + assertTrue(json.contains("\"name\":\"GET\""), json); + assertTrue(json.contains("\"kind\":3"), json); + } + + @Test + void nothingIsAddedWhenTelemetryIsOff() throws Exception { + assertFalse(Telemetry.isInstalled()); + NetworkManager.getInstance().addToQueueAndWait(request(API)); + assertNull(connection(API).getHeaders().get("traceparent")); + TelemetrySpan span = Telemetry.startSpan("noop"); + assertFalse(span.isRecording()); + assertNull(span.getTraceparent(), + "a span with no trace must not hand out an all-zero traceparent"); + span.end(); + } + + @Test + void anUnsampledTraceStillPropagatesItsDecision() throws Exception { + Telemetry.install(new TelemetryConfig().direct("http://collector.test").sampleRatio(0)); + NetworkManager.getInstance().addToQueueAndWait(request(API)); + String traceparent = connection(API).getHeaders().get("traceparent"); + assertNotNull(traceparent); + assertTrue(traceparent.endsWith("-00"), + "the backend must be told not to record either: " + traceparent); + TelemetrySpan unsampled = Telemetry.startSpan("unsampled"); + assertNotNull(unsampled.getTraceparent(), "an unsampled trace still propagates"); + unsampled.end(); + } + + @Test + void consentWhenAskedForGatesTracingEntirely() throws Exception { + AnalyticsConsent before = Analytics.getConsent(); + ConsentMode mode = Analytics.getConsentMode(); + try { + Analytics.setConsentMode(ConsentMode.OPT_IN); + Analytics.setConsent(null); + Telemetry.install(new TelemetryConfig().direct("http://collector.test") + .requireAnalyticsConsent(true)); + NetworkManager.getInstance().addToQueueAndWait(request(API + "?before")); + assertNull(connection(API + "?before").getHeaders().get("traceparent"), + "no consent yet under OPT_IN: the request must go out untouched"); + assertFalse(Telemetry.startSpan("x").isRecording()); + + Analytics.setConsent(AnalyticsConsent.granted()); + NetworkManager.getInstance().addToQueueAndWait(request(API + "?after")); + assertNotNull(connection(API + "?after").getHeaders().get("traceparent"), + "consent granted: the next request is traced"); + } finally { + Analytics.setConsent(before); + Analytics.setConsentMode(mode); + } + } + + @Test + void aQueuedExportStopsWhenConsentIsWithdrawn() throws Exception { + AnalyticsConsent before = Analytics.getConsent(); + try { + Analytics.setConsent(AnalyticsConsent.granted()); + Telemetry.State gated = new Telemetry.State(new TelemetryConfig() + .direct("http://collector.test").requireAnalyticsConsent(true)); + Telemetry.ExportRequest export = new Telemetry.ExportRequest(gated, new byte[] {1}); + assertFalse(export.shouldStop(), "with consent the export goes"); + Analytics.setConsent(AnalyticsConsent.denied()); + assertTrue(export.shouldStop(), + "an export queued before consent was withdrawn must not be sent"); + + // The export answers to the installation that recorded it, not to + // whatever is installed when it runs: neither nothing, nor a + // replacement that does not ask for consent. + Telemetry.uninstall(); + assertTrue(export.shouldStop(), "uninstalling released a consent-gated export"); + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + assertTrue(export.shouldStop(), + "an ungated reinstall released a consent-gated export"); + } finally { + Analytics.setConsent(before); + } + } + + @Test + void withoutTheFlagConsentIsNotConsulted() throws Exception { + AnalyticsConsent before = Analytics.getConsent(); + try { + Analytics.setConsent(AnalyticsConsent.denied()); + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + NetworkManager.getInstance().addToQueueAndWait(request(API + "?noflag")); + assertNotNull(connection(API + "?noflag").getHeaders().get("traceparent")); + } finally { + Analytics.setConsent(before); + } + } + + @Test + void aRedirectIsTwoAttemptsEachWithItsOwnStatusAndHeader() throws Exception { + // The first attempt answers 302. It returns before the guard's capture + // runs, and was reported as "no response"; and the request object is + // reused for the second attempt, which must carry ITS span, not the + // first attempt's header left behind. + TestCodenameOneImplementation impl = TestCodenameOneImplementation.getInstance(); + TestCodenameOneImplementation.TestConnection hop = impl.createConnection("http://hop.test/a"); + hop.setResponseCode(302); + hop.setHeader("location", API + "?hopped"); + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + NetworkManager.getInstance().addToQueueAndWait(request("http://hop.test/a")); + String second = connection(API + "?hopped").getHeaders().get("traceparent"); + assertNotNull(second, "the redirected attempt was not traced"); + + Telemetry.flush(); + List spans = exported(2); + Span redirect = null; + Span landed = null; + for (Span span : spans) { + String status = attribute(span.getAttributesList(), "http.response.status_code"); + if ("302".equals(status)) { + redirect = span; + } else if ("200".equals(status)) { + landed = span; + } + } + assertNotNull(redirect, "the 302 attempt must report its status: " + spans); + assertNotNull(landed, "the attempt the redirect reached: " + spans); + assertEquals(hex(landed.getSpanId()), second.substring(36, 52), + "the second attempt carried its own span, not the first's header"); + assertFalse(hex(redirect.getSpanId()).equals(hex(landed.getSpanId()))); + // Queued outside any action, yet one logical request: the second attempt + // continues the first attempt's trace, as its child, rather than a new one. + assertEquals(hex(redirect.getTraceId()), hex(landed.getTraceId()), + "a redirect split the request across two traces"); + assertEquals(hex(redirect.getSpanId()), hex(landed.getParentSpanId())); + } + + @Test + void anAppsOwnTraceparentIsNeverReplacedAndOursDoesNotOutliveTheAttempt() throws Exception { + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + String mine = "00-11111111111111111111111111111111-2222222222222222-01"; + ConnectionRequest own = request(API + "/own"); + own.addRequestHeader("Traceparent", mine); + NetworkManager.getInstance().addToQueueAndWait(own); + assertEquals(mine, connection(API + "/own").getHeaders().get("Traceparent")); + assertNull(connection(API + "/own").getHeaders().get("traceparent"), + "a second spelling of the header was added beside the app's"); + + ConnectionRequest ours = request(API + "/ours"); + NetworkManager.getInstance().addToQueueAndWait(ours); + assertNotNull(connection(API + "/ours").getHeaders().get("traceparent")); + assertTrue(ours.addRequestHeaderIfAbsent("traceparent", "x"), + "the tracer's header must be taken off the request when the attempt ends"); + + // The app's request joins the app's trace downstream, so no span of ours + // may describe it in another one; ours is recorded as usual. + Telemetry.flush(); + List spans = exported(1); + boolean sawOurs = false; + for (Span span : spans) { + String url = attribute(span.getAttributesList(), "url.full"); + assertFalse((API + "/own").equals(url), + "a span was recorded for a request that carries the app's own trace"); + sawOurs |= (API + "/ours").equals(url); + } + assertTrue(sawOurs, "the ordinary request's span is missing: " + spans); + } + + @Test + void theCurrentSpanBelongsToTheThreadThatStartedIt() throws Exception { + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + final TelemetrySpan[] seenElsewhere = new TelemetrySpan[] {Telemetry.startSpan("sentinel")}; + Telemetry.run("action", new Runnable() { + @Override + public void run() { + Thread other = new Thread(new Runnable() { + @Override + public void run() { + seenElsewhere[0] = Telemetry.getCurrentSpan(); + } + }); + other.start(); + try { + other.join(); + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + } + assertNotNull(Telemetry.getCurrentSpan()); + } + }); + assertNull(seenElsewhere[0], "another thread saw this thread's action as its own"); + } + + @Test + void anAttemptIsEndedByTheTracerThatStartedIt() throws Exception { + // The slot can be emptied while an attempt is in flight; the attempt must + // still be ended, and by its own tracer. + final int[] ended = new int[1]; + NetworkManager.setNetworkTracer(new com.codename1.io.NetworkTracer() { + @Override + public Object requestQueued(ConnectionRequest request) { + return null; + } + + @Override + public Object beforeRequest(ConnectionRequest request, Object parent) { + NetworkManager.setNetworkTracer(null); + return "attempt"; + } + + @Override + public void afterRequest(ConnectionRequest request, Object attempt, int status, + Throwable error) { + if ("attempt".equals(attempt)) { + ended[0]++; + } + } + }); + try { + NetworkManager.getInstance().addToQueueAndWait(request(API + "?swap")); + } finally { + NetworkManager.setNetworkTracer(null); + } + assertEquals(1, ended[0]); + } + + @Test + void aQueuedParentGoesOnlyToTheTracerThatCapturedIt() throws Exception { + // Swapped between queueing and running: the new tracer must not be handed + // the old one's opaque context. + final Object[] handed = new Object[] {"unset"}; + final com.codename1.io.NetworkTracer second = new com.codename1.io.NetworkTracer() { + @Override + public Object requestQueued(ConnectionRequest request) { + return null; + } + + @Override + public Object beforeRequest(ConnectionRequest request, Object parent) { + handed[0] = parent; + return null; + } + + @Override + public void afterRequest(ConnectionRequest request, Object attempt, int status, + Throwable error) { + } + }; + NetworkManager.setNetworkTracer(new com.codename1.io.NetworkTracer() { + @Override + public Object requestQueued(ConnectionRequest request) { + NetworkManager.setNetworkTracer(second); + return "the first tracer's context"; + } + + @Override + public Object beforeRequest(ConnectionRequest request, Object parent) { + return null; + } + + @Override + public void afterRequest(ConnectionRequest request, Object attempt, int status, + Throwable error) { + } + }); + try { + NetworkManager.getInstance().addToQueueAndWait(request(API + "?handover")); + } finally { + NetworkManager.setNetworkTracer(null); + } + assertNull(handed[0], "another tracer's context was passed on"); + } + + @Test + void aTrailingSlashDoesNotDuplicateTheTracesPath() { + assertEquals("https://c.test/v1/traces", + new TelemetryConfig().direct("https://c.test/v1/traces/").exportUrl()); + assertEquals("https://c.test/v1/traces", + new TelemetryConfig().direct("https://c.test//").exportUrl()); + assertEquals("https://api.test/otel/v1/traces", + new TelemetryConfig().relay("https://api.test/otel/v1/traces/").exportUrl()); + assertEquals("https://api.test/otel/v1/traces", + new TelemetryConfig().relay("https://api.test/").exportUrl()); + // A query carries the collector's key: the path goes BEFORE it. + assertEquals("https://c.test/otlp/v1/traces?api-key=s3cret", + new TelemetryConfig().direct("https://c.test/otlp?api-key=s3cret").exportUrl()); + assertEquals("https://c.test/v1/traces?api-key=s3cret", + new TelemetryConfig().direct("https://c.test/v1/traces/?api-key=s3cret").exportUrl()); + } + + @Test + void whileExportsAreBackedUpNoMoreAreQueuedAndTheBufferIsBounded() throws Exception { + // A queue that already holds the maximum of this installation's exports. + final Telemetry.State backedUp = new Telemetry.State( + new TelemetryConfig().direct("http://collector.test")) { + @Override + int pendingExports() { + return Telemetry.MAX_PENDING_EXPORTS; + } + }; + TestCodenameOneImplementation impl = TestCodenameOneImplementation.getInstance(); + impl.clearQueuedRequests(); + com.codename1.ui.CN.callSeriallyAndWait(new Runnable() { + @Override + public void run() { + for (int i = 0; i < 1000; i++) { + backedUp.record(backedUp.start("s" + i, TelemetrySpan.KIND_INTERNAL, null)); + } + } + }); + try { + for (ConnectionRequest queued : impl.getQueuedRequests()) { + assertFalse(queued instanceof Telemetry.ExportRequest, + "an export was queued behind the ones already waiting"); + } + // 32 per batch, so the bound is the 128 floor; the newest spans are kept. + assertEquals(128, backedUp.buffer.size(), "the buffer grew past its bound"); + assertEquals("s999", backedUp.buffer.get(backedUp.buffer.size() - 1).getName()); + } finally { + // Its flush timer started with the first span; it is never installed, + // so nothing else would stop it. + com.codename1.ui.CN.callSeriallyAndWait(new Runnable() { + @Override + public void run() { + backedUp.stop(); + } + }); + } + } + + @Test + void uninstallLeavesAReplacementTracerInPlace() { + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + com.codename1.io.NetworkTracer mine = new com.codename1.io.NetworkTracer() { + @Override + public Object requestQueued(ConnectionRequest request) { + return null; + } + + @Override + public Object beforeRequest(ConnectionRequest request, Object parent) { + return null; + } + + @Override + public void afterRequest(ConnectionRequest request, Object attempt, int status, + Throwable error) { + } + }; + NetworkManager.setNetworkTracer(mine); + try { + Telemetry.uninstall(); + assertTrue(NetworkManager.getNetworkTracer() == mine, + "uninstalling telemetry switched off the app's own tracer"); + } finally { + NetworkManager.setNetworkTracer(null); + } + } + + @Test + void anExportIsShortAndBehindTheAppsOwnRequests() throws Exception { + TestCodenameOneImplementation impl = TestCodenameOneImplementation.getInstance(); + impl.clearQueuedRequests(); + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + NetworkManager.getInstance().addToQueueAndWait(request(API + "/timed")); + Telemetry.flush(); + awaitConnection(COLLECTOR); + Telemetry.ExportRequest export = null; + for (ConnectionRequest queued : impl.getQueuedRequests()) { + if (queued instanceof Telemetry.ExportRequest) { + export = (Telemetry.ExportRequest) queued; + } + } + assertNotNull(export, "no export was queued"); + assertEquals(10000, export.getTimeout()); + assertEquals(10000, export.getReadTimeout()); + assertEquals(ConnectionRequest.PRIORITY_LOW, export.getPriority()); + } + + @Test + void onTheWebARelativeUrlIsSameOriginAndCarriesTheContext() throws Exception { + assertTrue(Telemetry.isSameOriginRelative("/api/orders")); + assertTrue(Telemetry.isSameOriginRelative("orders?next=http://x.test/")); + assertFalse(Telemetry.isSameOriginRelative("//other.test/api")); + // A browser reads '\\' as '/', so these are network paths too. + assertFalse(Telemetry.isSameOriginRelative("\\\\other.test/api")); + assertFalse(Telemetry.isSameOriginRelative("/\\other.test/api")); + assertFalse(Telemetry.isSameOriginRelative("\\/other.test/api")); + assertTrue(Telemetry.isSameOriginRelative("\\api\\orders")); + // The browser strips leading whitespace and controls, and every tab and + // newline, before it parses: these are network paths to it. + assertFalse(Telemetry.isSameOriginRelative(" //other.test/api")); + assertFalse(Telemetry.isSameOriginRelative("\t\\\\other.test/api")); + assertFalse(Telemetry.isSameOriginRelative("/\n/other.test/api")); + assertTrue(Telemetry.isSameOriginRelative(" /api/orders")); + assertFalse(Telemetry.isSameOriginRelative("http://other.test/api")); + assertFalse(Telemetry.isSameOriginRelative("data:text/plain,x")); + assertFalse(Telemetry.isSameOriginRelative("")); + + TestCodenameOneImplementation impl = TestCodenameOneImplementation.getInstance(); + impl.addNetworkMockResponse("/api/orders", 200, "OK", new byte[0]); + impl.addNetworkMockResponse("http://other.test/api", 200, "OK", new byte[0]); + impl.setPlatformName("HTML5"); + try { + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + // validate() refuses a relative URL; a request that sends one to its + // own origin has to relax it. + ConnectionRequest relative = new ConnectionRequest() { + @Override + protected void validate() { + } + + @Override + protected void readResponse(InputStream input) { + } + }; + relative.setUrl("/api/orders"); + relative.setPost(false); + NetworkManager.getInstance().addToQueueAndWait(relative); + assertNotNull(connection("/api/orders").getHeaders().get("traceparent"), + "a same-origin request lost its trace context on the web"); + NetworkManager.getInstance().addToQueueAndWait(request("http://other.test/api")); + assertNull(connection("http://other.test/api").getHeaders().get("traceparent"), + "a cross-origin request outside the allowlist got the header"); + } finally { + impl.setPlatformName(null); + } + } + + @Test + void aFailedAttemptThatIsRetriedStillReportsItsFailure() throws Exception { + // The retry re-queues the request from inside the exception handler; the + // failed attempt must be ended with its exception before that happens. + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + final int[] reads = new int[1]; + final int[] handled = new int[1]; + ConnectionRequest flaky = new ConnectionRequest() { + @Override + protected void readResponse(InputStream input) throws java.io.IOException { + if (reads[0]++ == 0) { + throw new java.io.IOException("connection reset"); + } + } + + @Override + protected void handleIOException(java.io.IOException err) { + handled[0]++; + retry(); + } + }; + flaky.setUrl(API + "?flaky"); + flaky.setPost(false); + NetworkManager.getInstance().addToQueue(flaky); + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (reads[0] < 2 && System.currentTimeMillis() < deadline) { + flushSerialCalls(); + Thread.sleep(20); + } + java.lang.reflect.Field errors = NetworkManager.class.getDeclaredField("errorListeners"); + errors.setAccessible(true); + assertEquals(2, reads[0], "the request was not retried: handleIOException ran " + + handled[0] + "x, global error listeners=" + + errors.get(NetworkManager.getInstance())); + + List spans = exported(2); + Span failed = null; + for (Span span : spans) { + if (span.getStatus().getCode() + == io.opentelemetry.proto.trace.v1.Status.StatusCode.STATUS_CODE_ERROR) { + failed = span; + } + } + assertNotNull(failed, "the failed attempt was exported as neither failed nor answered: " + + spans); + assertEquals("connection reset", failed.getStatus().getMessage()); + assertEquals("exception", failed.getEvents(0).getName()); + } + + @Test + void aBackslashEndsTheAuthorityAsItDoesInABrowser() { + String url = "https://evil.example\\@api.example/x"; + assertEquals("evil.example", Telemetry.host(url)); + assertEquals("https://evil.example:443", Telemetry.origin(url)); + assertEquals("https://evil.example:8443", + Telemetry.origin("https://evil.example:8443\\@api.example/x")); + } + + @Test + void redactionRemovesUserinfoThroughTheLastAt() { + assertEquals("https://host.example/x", + Telemetry.redact("https://alice:secret@tenant@host.example/x?q=1")); + assertEquals("https://host.example", + Telemetry.redact("https://a@b@host.example")); + // An '@' past the authority is path, not userinfo, and stays. + assertEquals("https://host.example/a@b", + Telemetry.redact("https://host.example/a@b")); + // A backslash does not end the authority for redaction: on a port whose + // URL parser keeps it in the authority, what precedes the '@' is userinfo. + assertEquals("https://b", Telemetry.redact("https://host.example\\a@b")); + } + + @Test + void aRelayTokenNoHeaderCouldCarryIsRefused() { + String[] bad = {"zq9\n", " zq9", "zq9 ", "zq9\t", "zq\u00019"}; + for (String token : bad) { + try { + new TelemetryConfig().relay("https://api.test").relayToken(token); + fail("accepted a relay token a header cannot carry"); + } catch (IllegalArgumentException expected) { + assertFalse(expected.getMessage().contains("zq"), expected.getMessage()); + } + } + new TelemetryConfig().relay("https://api.test").relayToken("t o\tk"); + } + + @Test + void anOversizedAttributeKeyIsDroppedNotCut() { + Telemetry.State state = new Telemetry.State( + new TelemetryConfig().direct("http://collector.test")); + TelemetrySpan span = state.start("keys", TelemetrySpan.KIND_INTERNAL, null); + StringBuilder huge = new StringBuilder(); + while (huge.length() <= TelemetrySpan.MAX_KEY_LENGTH) { + huge.append("key."); + } + span.setAttribute(huge.toString(), "a"); + span.setAttribute(huge.toString() + "other", 1L); + span.setAttribute("short", true); + assertEquals(1, span.attributes.size(), "an oversized key was kept: " + span.attributes); + assertEquals(2, span.droppedAttributes); + } + + @Test + void aSpanFromAnotherInstallationIsNeverAParent() { + Telemetry.State before = new Telemetry.State( + new TelemetryConfig().direct("http://collector.test")); + Telemetry.State after = new Telemetry.State( + new TelemetryConfig().direct("http://collector.test")); + TelemetrySpan old = before.start("old action", TelemetrySpan.KIND_INTERNAL, null); + TelemetrySpan mine = after.start("request", TelemetrySpan.KIND_CLIENT, old); + TelemetrySpan child = after.start("child", TelemetrySpan.KIND_CLIENT, mine); + assertNotNull(old); + assertNotNull(mine); + assertFalse(old.getTraceId().equals(mine.getTraceId()), + "a new installation joined the previous one's trace"); + assertNull(mine.parentSpanId); + assertEquals(mine.getTraceId(), child.getTraceId(), "its own spans still nest"); + assertEquals(mine.getSpanId(), child.parentSpanId); + } + + @Test + void anEndpointNoExportCouldReachIsRefusedWhenGiven() { + String[] bad = {"https://", "https:///v1/traces", "ftp://collector.test", + "collector.test:4318", "https://collector example", "https://c.test:99999", + "https://[nope]:4318", "https://bad value@collector.test", "https://u%zz@collector.test", + "https://u%2@collector.test", "https://collector.test/bad path", + "https://collector.test#api-key=s3cret"}; + for (String url : bad) { + try { + new TelemetryConfig().direct(url); + throw new AssertionError("accepted " + url); + } catch (IllegalArgumentException expected) { + // Refused at the call, not lost at the first export. + } + } + try { + new TelemetryConfig().relay("https://user:s3cret@bad host/"); + throw new AssertionError("accepted a host with a space"); + } catch (IllegalArgumentException expected) { + assertFalse(expected.getMessage().contains("s3cret"), + "the refusal quoted a credential: " + expected.getMessage()); + } + assertNull(new TelemetryConfig().direct(null).exportUrl(), "no endpoint is still allowed"); + assertNull(new TelemetryConfig().relay("").exportUrl()); + assertEquals("https://[::1]:4318/v1/traces", + new TelemetryConfig().direct("https://[::1]:4318").exportUrl()); + assertNotNull(new TelemetryConfig().direct("https://user:p%40ss@collector.test").exportUrl(), + "valid userinfo, a percent escape included, is accepted"); + assertFalse(TelemetryConfig.isHttpUrl("https://[:::]:4318")); + assertFalse(TelemetryConfig.isHttpUrl("https://[1::2::3]:4318")); + assertFalse(TelemetryConfig.isHttpUrl("https://[1:2:3:4:5:6:7:8:9]:4318")); + assertFalse(TelemetryConfig.isHttpUrl("https://[::ffff:300.0.0.1]:4318")); + assertFalse(TelemetryConfig.isHttpUrl("https://[1:2:3:4:5:6:7]:4318")); + assertFalse(TelemetryConfig.isHttpUrl("https://[12345::1]:4318")); + assertTrue(TelemetryConfig.isHttpUrl("https://[::1]:4318")); + assertTrue(TelemetryConfig.isHttpUrl("https://[::ffff:10.0.0.7]:4318")); + assertTrue(TelemetryConfig.isHttpUrl("https://[2001:db8::1]:4318")); + assertTrue(TelemetryConfig.isHttpUrl("https://[1:2:3:4:5:6:7:8]:4318")); + assertTrue(TelemetryConfig.isHttpUrl("https://[::]:4318")); + } + + @Test + void aHeaderOrRatioNoExportCouldUseIsRefusedWhenGiven() { + String[][] bad = {{"Authorization", "token\nextra"}, {"Bad Name", "x"}, {"", "x"}, + {"Content-Type", "application/json"}, {"content-length", "3"}, + {"X-Key", "a\u0000b"}, {"X-Key", "a\u007fb"}}; + for (String[] header : bad) { + try { + new TelemetryConfig().header(header[0], header[1]); + throw new AssertionError("accepted header '" + header[0] + "'"); + } catch (IllegalArgumentException expected) { + assertFalse(expected.getMessage().contains("token\nextra"), + "the refusal quoted the value: " + expected.getMessage()); + } + } + new TelemetryConfig().header("Authorization", "Api-Token\tabc"); + try { + new TelemetryConfig().relayToken("r3lay\r\nX-Evil: 1"); + throw new AssertionError("accepted a relay token with a line break"); + } catch (IllegalArgumentException expected) { + // Sent as a header, so held to the same rules. + } + try { + new TelemetryConfig().sampleRatio(Double.NaN); + throw new AssertionError("accepted a NaN ratio"); + } catch (IllegalArgumentException expected) { + // A range clamps; NaN has no nearest value. + } + assertEquals(1.0, new TelemetryConfig().sampleRatio(7).sampleRatio, 0); + } + + @Test + void theRelayIsMatchedByOriginNotHost() { + assertEquals("https://api.example.com:443", Telemetry.origin("https://API.example.com/x")); + assertEquals("https://api.example.com:8443", + Telemetry.origin("https://u:p@api.example.com:8443/otel?q=1")); + assertEquals("http://api.example.com:80", Telemetry.origin("http://api.example.com")); + assertEquals("http://[::1]:4318", Telemetry.origin("http://[::1]:4318/v1")); + assertNull(Telemetry.origin("/relative")); + + TestCodenameOneImplementation impl = TestCodenameOneImplementation.getInstance(); + String same = "https://api.example.com:8443/pets"; + String otherPort = "https://api.example.com:9443/pets"; + String otherScheme = "http://api.example.com/pets"; + impl.addNetworkMockResponse(same, 200, "OK", new byte[0]); + impl.addNetworkMockResponse(otherPort, 200, "OK", new byte[0]); + impl.addNetworkMockResponse(otherScheme, 200, "OK", new byte[0]); + impl.setPlatformName("HTML5"); + try { + Telemetry.install(new TelemetryConfig().relay("https://api.example.com:8443")); + NetworkManager.getInstance().addToQueueAndWait(request(same)); + NetworkManager.getInstance().addToQueueAndWait(request(otherPort)); + NetworkManager.getInstance().addToQueueAndWait(request(otherScheme)); + assertNotNull(connection(same).getHeaders().get("traceparent"), + "the relay's own origin carries the context"); + assertNull(connection(otherPort).getHeaders().get("traceparent"), + "another port on the relay's host is another origin"); + assertNull(connection(otherScheme).getHeaders().get("traceparent"), + "another scheme on the relay's host is another origin"); + } finally { + impl.setPlatformName(null); + } + } + + @Test + void theInstalledConfigurationIsASnapshot() throws Exception { + // Every field, by reflection, so a field added later without being copied + // -- or without being set here -- fails this test instead of being read + // live from the caller's object again. + TelemetryConfig original = new TelemetryConfig().direct("https://c.test") + .serviceName("svc").relayToken("tok").protobuf(false).sampleRatio(0.5) + .batchSize(7).flushIntervalMillis(1234).header("X-A", "1") + .propagateTo("h.test").propagateToAllHosts().requireAnalyticsConsent(true); + TelemetryConfig copy = original.copy(); + TelemetryConfig defaults = new TelemetryConfig(); + for (java.lang.reflect.Field field : TelemetryConfig.class.getDeclaredFields()) { + if (java.lang.reflect.Modifier.isStatic(field.getModifiers()) || field.isSynthetic()) { + continue; + } + field.setAccessible(true); + Object set = field.get(original); + Object copied = field.get(copy); + Object unset = field.get(defaults); + if (set instanceof List) { + assertFalse(set == copied, field.getName() + " is shared, not copied"); + assertEquals(((List) set).size(), ((List) copied).size(), field.getName()); + assertFalse(((List) set).isEmpty(), field.getName() + " was not set by this test"); + continue; + } + assertEquals(set, copied, field.getName() + " was not copied"); + assertFalse(set == null ? unset == null : set.equals(unset), + field.getName() + " was not set by this test, so its copy is unchecked"); + } + + // And in use: installed as direct protobuf, then the caller's object is + // turned into a JSON relay. The installation must not notice. + TelemetryConfig live = new TelemetryConfig().direct("http://collector.test"); + Telemetry.install(live); + live.relay("http://backend.test").protobuf(false); + NetworkManager.getInstance().addToQueueAndWait(request(API + "?snapshot")); + Telemetry.flush(); + assertFalse(exported(1).isEmpty(), + "the export was not the protobuf the installed configuration asked for"); + } + + @Test + void anAttemptCancelledAfterItStartedIsAnError() throws Exception { + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + final int[] checks = new int[1]; + ConnectionRequest cancelled = new ConnectionRequest() { + @Override + protected boolean shouldStop() { + // Running at the first check, stopped at the next: the span has + // started and no response ever arrives. + return checks[0]++ > 0; + } + + @Override + protected void readResponse(InputStream input) { + } + }; + cancelled.setUrl(API + "?cancelled"); + cancelled.setPost(false); + NetworkManager.getInstance().addToQueueAndWait(cancelled); + Telemetry.flush(); + Span span = find(exported(1), "GET"); + assertEquals(io.opentelemetry.proto.trace.v1.Status.StatusCode.STATUS_CODE_ERROR, + span.getStatus().getCode(), "a cancelled attempt read as a success"); + assertEquals("cancelled before a response", span.getStatus().getMessage()); + } + + @Test + void anExportIsNeverRedirectedWithItsCredential() throws Exception { + TestCodenameOneImplementation impl = TestCodenameOneImplementation.getInstance(); + TestCodenameOneImplementation.TestConnection redirecting = + impl.createConnection("http://redirecting.test/v1/traces"); + redirecting.setResponseCode(302); + redirecting.setHeader("location", "http://elsewhere.test/v1/traces"); + impl.addNetworkMockResponse("http://elsewhere.test/v1/traces", 200, "OK", new byte[0]); + Telemetry.install(new TelemetryConfig().direct("http://redirecting.test") + .header("Authorization", "Api-Token s3cret")); + NetworkManager.getInstance().addToQueueAndWait(request(API + "?redirected")); + Telemetry.flush(); + // An upper bound, not a delay: the loop leaves as soon as the export lands. + // Exports queue at low priority behind whatever else the shared network + // manager holds, and 3s was too short on a loaded CI runner. + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + TestCodenameOneImplementation.TestConnection sent = null; + while (System.currentTimeMillis() < deadline) { + flushSerialCalls(); + sent = impl.getConnection("http://redirecting.test/v1/traces"); + if (sent != null && sent.getOutputData().length > 0) { + break; + } + Thread.sleep(20); + } + assertTrue(sent != null && sent.getOutputData().length > 0, + "the export never reached the collector"); + assertNull(impl.getConnection("http://elsewhere.test/v1/traces"), + "the export followed a redirect, credential and all"); + } + + @Test + void aRetryFromAResponseCodeListenerContinuesTheTrace() throws Exception { + // The listener runs LATER, on the EDT, after the network thread has ended + // the attempt; its retry must still continue the failed attempt's trace. + TestCodenameOneImplementation impl = TestCodenameOneImplementation.getInstance(); + final TestCodenameOneImplementation.TestConnection mock = + impl.createConnection("http://flaky.test/api"); + mock.setResponseCode(503); + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + final int[] reads = new int[1]; + final ConnectionRequest flaky = new ConnectionRequest() { + @Override + protected void readResponse(InputStream input) { + reads[0]++; + } + }; + flaky.setUrl("http://flaky.test/api"); + flaky.setPost(false); + final boolean[] retried = new boolean[1]; + flaky.addResponseCodeListener(new com.codename1.ui.events.ActionListener() { + @Override + public void actionPerformed(com.codename1.io.NetworkEvent evt) { + if (!retried[0]) { + retried[0] = true; + mock.setResponseCode(200); + flaky.retry(); + } + } + }); + NetworkManager.getInstance().addToQueue(flaky); + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + // Both attempts read their response: the 503 too, since error bodies are + // read by default. + while (reads[0] < 2 && System.currentTimeMillis() < deadline) { + flushSerialCalls(); + Thread.sleep(20); + } + assertTrue(retried[0], "the listener never ran"); + assertEquals(2, reads[0], "the retry never completed"); + + List spans = exported(2); + Span failed = null; + Span landed = null; + for (Span span : spans) { + String status = attribute(span.getAttributesList(), "http.response.status_code"); + if ("503".equals(status)) { + failed = span; + } else if ("200".equals(status)) { + landed = span; + } + } + assertNotNull(failed, String.valueOf(spans)); + assertNotNull(landed, String.valueOf(spans)); + assertEquals(hex(failed.getTraceId()), hex(landed.getTraceId()), + "a listener's retry started an unrelated trace"); + assertEquals(hex(failed.getSpanId()), hex(landed.getParentSpanId())); + } + + @Test + void aBlankServiceNameFallsBackRatherThanExportingBlank() throws Exception { + Telemetry.install(new TelemetryConfig().direct("http://collector.test").serviceName(" ")); + NetworkManager.getInstance().addToQueueAndWait(request(API + "?blank-service")); + Telemetry.flush(); + exported(1); + ExportTraceServiceRequest sent = ExportTraceServiceRequest.parseFrom( + connection(COLLECTOR).getOutputData()); + String service = attribute(sent.getResourceSpans(0).getResource().getAttributesList(), + "service.name"); + assertNotNull(service); + assertFalse(service.trim().length() == 0, "a blank service.name was exported"); + } + + @Test + void stoppingBehindAFullExportQueueStillSendsTheBuffer() throws Exception { + // Uninstall while exports are already waiting: the final flush goes out + // anyway, or the buffered spans die with the installation. + final Telemetry.State backedUp = new Telemetry.State( + new TelemetryConfig().direct("http://collector.test")) { + @Override + int pendingExports() { + return Telemetry.MAX_PENDING_EXPORTS; + } + }; + TestCodenameOneImplementation impl = TestCodenameOneImplementation.getInstance(); + impl.clearQueuedRequests(); + com.codename1.ui.CN.callSeriallyAndWait(new Runnable() { + @Override + public void run() { + backedUp.record(backedUp.start("last words", TelemetrySpan.KIND_INTERNAL, null)); + backedUp.stop(); + } + }); + boolean exported = false; + for (ConnectionRequest queued : impl.getQueuedRequests()) { + exported |= queued instanceof Telemetry.ExportRequest; + } + assertTrue(exported, "the buffer was dropped on stop because the export queue was full"); + assertTrue(backedUp.buffer.isEmpty()); + } + + @Test + void aDefaultTraceparentIsHonouredAndNeverDuplicated() throws Exception { + // Defaults are copied onto the request before the tracer sees it, so a trace + // context the app supplies as a default is the app's choice of parent, and is + // never joined by a second spelling -- neither the tracer's nor the request's. + String mine = "00-33333333333333333333333333333333-4444444444444444-01"; + NetworkManager.getInstance().addDefaultHeader("Traceparent", mine); + NetworkManager.getInstance().addDefaultHeader("x-default-only", "d"); + try { + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + NetworkManager.getInstance().addToQueueAndWait(request(API + "/default-parent")); + Map sent = connection(API + "/default-parent").getHeaders(); + assertEquals(mine, sent.get("Traceparent")); + assertNull(sent.get("traceparent"), "the tracer added its own beside the default"); + + ConnectionRequest own = request(API + "/own-spelling"); + own.addRequestHeader("X-DEFAULT-ONLY", "r"); + NetworkManager.getInstance().addToQueueAndWait(own); + sent = connection(API + "/own-spelling").getHeaders(); + assertEquals("r", sent.get("X-DEFAULT-ONLY")); + assertNull(sent.get("x-default-only"), + "a default went out beside the request's own spelling of the header"); + } finally { + java.lang.reflect.Field headers = NetworkManager.class.getDeclaredField("userHeaders"); + headers.setAccessible(true); + headers.set(NetworkManager.getInstance(), null); + } + } + + @Test + void anExportNeverCarriesTheAppsDefaultHeaders() throws Exception { + // A default header is the app's credential for its own services; it must + // not reach a third-party collector, nor relabel the export's body. + NetworkManager.getInstance().addDefaultHeader("Authorization", "app-secret"); + NetworkManager.getInstance().addDefaultHeader("Content-Type", "text/plain"); + try { + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + NetworkManager.getInstance().addToQueueAndWait(request(API + "?defaults")); + assertEquals("app-secret", connection(API + "?defaults").getHeaders().get("Authorization"), + "the app's own request still gets its default header"); + Telemetry.flush(); + exported(1); + TestCodenameOneImplementation.TestConnection export = connection(COLLECTOR); + assertNull(export.getHeaders().get("Authorization"), + "the app's credential was sent to the collector"); + assertFalse("text/plain".equals(export.getHeaders().get("Content-Type")), + "a default Content-Type relabelled the export"); + } finally { + java.lang.reflect.Field headers = NetworkManager.class.getDeclaredField("userHeaders"); + headers.setAccessible(true); + headers.set(NetworkManager.getInstance(), null); + } + } + + @Test + void aSpanThatEndedJustBeforeStopIsStillExported() throws Exception { + // It ends on another thread, so it is handed to the EDT -- and the stop + // runs on the EDT first. It finished while telemetry ran, and is sent. + final Telemetry.State state = new Telemetry.State( + new TelemetryConfig().direct("http://collector.test")); + TestCodenameOneImplementation impl = TestCodenameOneImplementation.getInstance(); + impl.clearQueuedRequests(); + com.codename1.ui.CN.callSeriallyAndWait(new Runnable() { + @Override + public void run() { + final TelemetrySpan[] spans = new TelemetrySpan[5]; + for (int i = 0; i < spans.length; i++) { + spans[i] = state.start("late" + i, TelemetrySpan.KIND_CLIENT, null); + } + Thread ender = new Thread(new Runnable() { + @Override + public void run() { + for (TelemetrySpan span : spans) { + span.end(); + } + } + }); + ender.start(); + try { + ender.join(); + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + } + state.stop(); // before the handed-off span reaches the EDT + } + }); + // The handoffs and then the one export they share are each a turn of the + // EDT, so it takes more than one flush; wait for the export itself. + int exports = 0; + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (exports == 0 && System.currentTimeMillis() < deadline) { + flushSerialCalls(); + for (ConnectionRequest queued : impl.getQueuedRequests()) { + if (queued instanceof Telemetry.ExportRequest) { + exports++; + } + } + if (exports == 0) { + Thread.sleep(20); + } + } + assertTrue(exports > 0, "spans that ended before the stop were discarded after it"); + assertEquals(1, exports, "a burst of late spans went out as one export per span"); + } + + @Test + void aSpanNameIsBounded() { + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + StringBuilder huge = new StringBuilder(); + while (huge.length() < 50000) { + huge.append("name "); + } + TelemetrySpan span = Telemetry.startSpan(huge.toString()); + assertTrue(span.getName().length() <= TelemetrySpan.MAX_VALUE_LENGTH, "" + span.getName().length()); + span.updateName(huge.toString()); + assertTrue(span.getName().length() <= TelemetrySpan.MAX_VALUE_LENGTH); + span.end(); + } + + @Test + void aRequestCarryingOnlyATracestateIsTheAppsOwn() throws Exception { + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + ConnectionRequest own = request(API + "/state-only"); + own.addRequestHeader("tracestate", "vendor=opaque"); + NetworkManager.getInstance().addToQueueAndWait(own); + assertNull(connection(API + "/state-only").getHeaders().get("traceparent"), + "a traceparent of ours was paired with the app's tracestate"); + } + + @Test + void anExportKeepsNoneOfALargeAcknowledgement() throws Exception { + // A collector (or a proxy) answering 200 with a large body: the export + // reads it only to discard it, and holds none of it afterwards. + byte[] huge = new byte[1 << 20]; + TestCodenameOneImplementation impl = TestCodenameOneImplementation.getInstance(); + impl.addNetworkMockResponse("http://bigack.test/v1/traces", 200, "OK", huge); + impl.clearQueuedRequests(); + Telemetry.install(new TelemetryConfig().direct("http://bigack.test")); + NetworkManager.getInstance().addToQueueAndWait(request(API + "?bigack")); + Telemetry.flush(); + Telemetry.ExportRequest export = null; + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (export == null && System.currentTimeMillis() < deadline) { + flushSerialCalls(); + for (ConnectionRequest queued : impl.getQueuedRequests()) { + if (queued instanceof Telemetry.ExportRequest) { + export = (Telemetry.ExportRequest) queued; + } + } + Thread.sleep(20); + } + assertNotNull(export, "no export was queued"); + awaitConnection("http://bigack.test/v1/traces"); + // Its response handling runs after the body was written; give it the + // network thread's turn to finish before looking. + long settle = System.currentTimeMillis() + 2000; + while (System.currentTimeMillis() < settle && !export.isReadResponseForErrors() + && export.getResponseData() == null && export.getResponseCode() == 0) { + flushSerialCalls(); + Thread.sleep(20); + } + byte[] kept = export.getResponseData(); + assertTrue(kept == null, "the export kept the collector's body in memory: " + + (kept == null ? 0 : kept.length) + " bytes"); + } + + @Test + void aTruncatedValueNeverEndsInHalfACharacter() { + StringBuilder text = new StringBuilder(); + for (int i = 0; i < TelemetrySpan.MAX_VALUE_LENGTH - 1; i++) { + text.append('a'); + } + text.append("\ud83d\ude00tail"); + String bounded = TelemetrySpan.bound(text.toString()); + assertEquals(TelemetrySpan.MAX_VALUE_LENGTH - 1, bounded.length(), + "the pair straddling the limit must go whole"); + assertFalse(Character.isHighSurrogate(bounded.charAt(bounded.length() - 1))); + } + + @Test + void anExceptionStatusIsBounded() { + Telemetry.install(new TelemetryConfig().direct("http://collector.test")); + TelemetrySpan span = Telemetry.startSpan("big"); + StringBuilder huge = new StringBuilder(); + while (huge.length() < 20000) { + huge.append("0123456789"); + } + span.recordException(new RuntimeException(huge.toString())); + assertEquals(TelemetrySpan.MAX_VALUE_LENGTH, span.statusMessage.length()); + span.end(); + } + + // ------------------------------------------------------------------ + + private static ConnectionRequest request(String url) { + ConnectionRequest request = new ConnectionRequest() { + @Override + protected void readResponse(InputStream input) { + // The body is not what these tests are about. + } + }; + request.setUrl(url); + request.setPost(false); + return request; + } + + private static TestCodenameOneImplementation.TestConnection connection(String url) { + TestCodenameOneImplementation.TestConnection c = + TestCodenameOneImplementation.getInstance().getConnection(url); + assertNotNull(c, "no request was made to " + url); + return c; + } + + /// Waits for the export: the span reaches the buffer through the EDT, and the + /// export itself is queued behind it. + private TestCodenameOneImplementation.TestConnection awaitConnection(String url) + throws InterruptedException { + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (System.currentTimeMillis() < deadline) { + flushSerialCalls(); + TestCodenameOneImplementation.TestConnection c = + TestCodenameOneImplementation.getInstance().getConnection(url); + if (c != null && c.getOutputData().length > 0) { + return c; + } + Thread.sleep(20); + } + throw new AssertionError("nothing was exported to " + url); + } + + /// Every span the collector has received, once there are at least `wanted`. + private List exported(int wanted) throws Exception { + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + List spans = new ArrayList(); + while (System.currentTimeMillis() < deadline) { + flushSerialCalls(); + Telemetry.flush(); + TestCodenameOneImplementation.TestConnection c = + TestCodenameOneImplementation.getInstance().getConnection(COLLECTOR); + if (c != null && c.getOutputData().length > 0) { + // Every export to one URL writes into the same mock connection, + // so the bytes are several requests back to back. Protobuf + // messages concatenate by merging their repeated fields, which + // is exactly "every export's spans". + spans = new ArrayList(); + ExportTraceServiceRequest merged = ExportTraceServiceRequest.parseFrom( + c.getOutputData()); + for (int r = 0; r < merged.getResourceSpansCount(); r++) { + for (int s = 0; s < merged.getResourceSpans(r).getScopeSpansCount(); s++) { + spans.addAll(merged.getResourceSpans(r).getScopeSpans(s).getSpansList()); + } + } + assertEquals("com.codename1.telemetry", + merged.getResourceSpans(0).getScopeSpans(0).getScope().getName()); + if (spans.size() >= wanted) { + return spans; + } + } + Thread.sleep(20); + } + throw new AssertionError("expected " + wanted + " spans, got " + spans); + } + + private static Span find(List spans, String name) { + for (Span span : spans) { + if (name.equals(span.getName())) { + return span; + } + } + throw new AssertionError("no span named " + name + " in " + spans); + } + + private static String attribute(List attributes, String key) { + for (KeyValue kv : attributes) { + if (kv.getKey().equals(key)) { + return kv.getValue().hasIntValue() + ? String.valueOf(kv.getValue().getIntValue()) + : kv.getValue().getStringValue(); + } + } + return null; + } + + private static String hex(ByteString bytes) { + StringBuilder out = new StringBuilder(); + for (int i = 0; i < bytes.size(); i++) { + out.append(String.format("%02x", bytes.byteAt(i) & 0xff)); + } + return out.toString(); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java b/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java index 445d7a78bc6..34c24c90f94 100644 --- a/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java +++ b/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java @@ -1486,7 +1486,33 @@ public void clearFileSystem() { fileSystem.clear(); } + /** + * Receives every component a repaint is requested for, as it is requested, or + * null for none. A test that asks "was X queued for repaint" by reading the + * paint queue afterwards races the EDT: every flush paints, and painting + * empties that queue. Observing the request itself does not. + */ + private java.util.List repaintLog; + + public void recordRepaints(java.util.List log) { + this.repaintLog = log; + } + + @Override + public void repaint(com.codename1.ui.animations.Animation cmp) { + java.util.List log = repaintLog; + if (log != null) { + log.add(cmp); + } + super.repaint(cmp); + } + public void reset() { + repaintLog = null; + // Back to the default a test class starts from. Runs on every UITestBase + // teardown, so a test that switches touch off and forgets -- or restores the + // wrong value -- cannot decide the command behaviour of the classes after it. + touchDevice = true; minimized = false; usesInvokeAndBlockForEditString = false; windowManager = null; diff --git a/maven/core-unittests/src/test/java/com/codename1/ui/SheetSwipeToDismissTest.java b/maven/core-unittests/src/test/java/com/codename1/ui/SheetSwipeToDismissTest.java index 0d59b300446..bf67db72ecd 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ui/SheetSwipeToDismissTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ui/SheetSwipeToDismissTest.java @@ -248,6 +248,7 @@ void swipeDismissTriggersFormRepaint() throws Exception { int dragDistance = (int) (sheet.getHeight() * 0.6); dragSheet(x, startY, 0, dragDistance, 5); + java.util.List afterRemoval = recordRepaintsAfterRemoval(sheet); implementation.dispatchPointerRelease(x, startY + dragDistance); flushSerialCalls(); @@ -260,10 +261,9 @@ void swipeDismissTriggersFormRepaint() throws Exception { assertNull(Sheet.getCurrentSheet(), "Sanity: sheet must be dismissed before checking repaint state"); - assertPaintScheduledOrAnimating(form, + assertRepaintedAfterRemoval(form, afterRemoval, "After a swipe-to-dismiss the form must be queued for repaint " - + "(or another animation must keep the EDT awake) so " - + "the dim overlay is cleared. Otherwise the EDT " + + "so the dim overlay is cleared. Otherwise the EDT " + "idles with stale dim pixels until the user taps."); } @@ -281,6 +281,7 @@ void backButtonDismissTriggersFormRepaint() throws Exception { // does. With no parent sheet this routes to hide(duration), which // is the path the user reports as "working fine" -- the baseline // we want to compare swipe-dismiss against. + java.util.List afterRemoval = recordRepaintsAfterRemoval(sheet); sheet.back(300); flushSerialCalls(); assertTrue(form.getAnimationManager().isAnimating(), @@ -290,7 +291,7 @@ void backButtonDismissTriggersFormRepaint() throws Exception { assertNull(Sheet.getCurrentSheet(), "Sanity: sheet must be dismissed before checking repaint state"); - assertPaintScheduledOrAnimating(form, + assertRepaintedAfterRemoval(form, afterRemoval, "After a back-button dismiss the form must be queued for " + "repaint so the dim overlay is cleared"); } @@ -340,7 +341,15 @@ private void awaitAnimationsFlushingPaintQueue(Form form) throws Exception { // the next tick: when it flips false, the next updateAnimations // call is the one that runs the completion runnable. while (am.isAnimating() && System.currentTimeMillis() < deadline) { - clearPaintQueue(); + // Cleared only while the sheet is still up. Once the dismiss has run, + // the repaint it scheduled is the thing under test -- and another + // animation still finishing on the form (the overlay's fade, which on + // a slow runner can outlast the slide) kept this loop going and + // cleared that repaint away, so the assertion below found an empty + // queue on a JDK 21 CI run with the fix in place. + if (Sheet.getCurrentSheet() != null) { + clearPaintQueue(); + } am.updateAnimations(); flushSerialCalls(); sleepQuietly(10); @@ -380,35 +389,37 @@ private void clearPaintQueue() throws Exception { /// form, the content pane, or an ancestor of the content. Equivalently, /// if another animation is still running, a paint cycle will follow /// regardless and the assertion is satisfied. - private void assertPaintScheduledOrAnimating(Form form, String message) throws Exception { - AnimationManager am = form.getAnimationManager(); - if (am.isAnimating()) { - return; - } - Class implClass = Class.forName("com.codename1.impl.CodenameOneImplementation"); - Field mainSurfaceField = implClass.getDeclaredField("mainSurface"); - mainSurfaceField.setAccessible(true); - Object surface = mainSurfaceField.get(implementation); - Class surfaceClass = surface.getClass(); - Field fillField = surfaceClass.getDeclaredField("paintQueueFill"); - Field queueField = surfaceClass.getDeclaredField("paintQueue"); - fillField.setAccessible(true); - queueField.setAccessible(true); - int fill = fillField.getInt(surface); - Object queue = queueField.get(surface); + /// Records the components a repaint is requested for once `sheet` has left + /// the form -- the dismiss path's own repaint, which is what #4899 is about. + /// Observed as requested rather than read from the paint queue afterwards: + /// every flush paints and so empties that queue, and a snapshot of it failed + /// on CI runners with the fix in place (paintQueue=[]). Mid-animation + /// repaints, made while the sheet is still attached, are not counted. + private java.util.List recordRepaintsAfterRemoval(final Sheet sheet) { + final java.util.List afterRemoval = new java.util.ArrayList(); + implementation.recordRepaints(new java.util.ArrayList() { + @Override + public boolean add(Object cmp) { + if (sheet.getComponentForm() == null) { + afterRemoval.add(cmp); + } + return true; + } + }); + return afterRemoval; + } + private void assertRepaintedAfterRemoval(Form form, java.util.List repainted, + String message) { + implementation.recordRepaints(null); Component content = form.getContentPane(); boolean covered = false; StringBuilder seen = new StringBuilder("["); - for (int i = 0; i < fill; i++) { - Object entry = java.lang.reflect.Array.get(queue, i); - if (entry == null) { - continue; - } + for (Object entry : repainted) { if (seen.length() > 1) { seen.append(", "); } - seen.append(entry.getClass().getSimpleName()); + seen.append(entry == null ? "null" : entry.getClass().getSimpleName()); if (entry == form || entry == content) { covered = true; } else if (entry instanceof Container && ((Container) entry).contains(content)) { @@ -416,7 +427,7 @@ private void assertPaintScheduledOrAnimating(Form form, String message) throws E } } seen.append("]"); - assertTrue(covered, message + ". paintQueue=" + seen); + assertTrue(covered, message + ". repainted after removal=" + seen); } private Form showFormWithSheet(String title) { diff --git a/scripts/ci/retry.sh b/scripts/ci/retry.sh index f3fd573a4c4..cbb1393b66d 100755 --- a/scripts/ci/retry.sh +++ b/scripts/ci/retry.sh @@ -35,7 +35,11 @@ set -uo pipefail # "or one of its dependencies could not be resolved" -- a plain Central hiccup that # two other workflows already knew to retry and that one had never been taught. One # definition, so the next failure mode is added once. -TRANSIENT_RESOLUTION_FAILURE='status: (403|429|50[0-9])|Could not transfer artifact|Could not resolve dependencies|Failed to read artifact descriptor|Unresolveable build extension|Non-resolvable import POM|or one of its dependencies could not be resolved|authorization failed for https://repo\.maven\.apache\.org|Connection reset|Premature end of Content-Length' +# UnknownHostException and the curl/glibc spellings of the same thing: a DNS +# failure on the runner is as transient as a Central 5xx. A windows cross-build +# died on "UnknownHostException: raw.githubusercontent.com" from an Ant , +# whose own retries all land inside the same second. +TRANSIENT_RESOLUTION_FAILURE='status: (403|429|50[0-9])|Could not transfer artifact|Could not resolve dependencies|Failed to read artifact descriptor|Unresolveable build extension|Non-resolvable import POM|or one of its dependencies could not be resolved|authorization failed for https://repo\.maven\.apache\.org|Connection reset|Premature end of Content-Length|UnknownHostException|Could not resolve host|Temporary failure in name resolution' # Four, not three. With the growing wait below that is 30s, 2m and 5m of # coverage -- the same 30/120/300 the Windows cross-compile workflow settled on. @@ -53,6 +57,55 @@ if [ "$only_matching" = "transient" ]; then only_matching="$TRANSIENT_RESOLUTION_FAILURE" fi +# Maven records a download that failed as a `*.lastUpdated` marker beside the +# artifact, and then answers later resolutions from that marker instead of the +# network: "was not found ... during a previous attempt. This failure was +# cached in the local repository". A retry that leaves the marker in place is +# therefore guaranteed to fail the same way -- which is how a single Central +# hiccup took four attempts and the job with it, and, because `cache: maven` +# stores the local repository between runs, went on failing runs that Central +# would have answered fine. +# +# Clearing them before each retry is safe: the marker records only a FAILURE, +# never a downloaded artifact, so the worst case is that Maven asks Central a +# question it already knows the answer to. +purge_maven_negative_cache() { + # EVERY place the local repository can be, not just $HOME/.m2. In a GitHub + # container job $HOME is /github/home, but Maven takes its repository from + # Java's user.home, which JDK 8 reads from the passwd entry: /root. Measured + # with eclipse-temurin:8 as root and HOME=/github/home -- user.home = /root. + # Looking only under $HOME found nothing, said nothing, and a purchase-e2e run + # replayed one cached miss through all five attempts. + repos="" + for arg in "$@" ${MAVEN_OPTS:-}; do + case "$arg" in + -Dmaven.repo.local=*) repos="$repos ${arg#-Dmaven.repo.local=}" ;; + esac + done + [ -n "${MAVEN_REPO_LOCAL:-}" ] && repos="$repos ${MAVEN_REPO_LOCAL}" + repos="$repos ${HOME:-}/.m2/repository" + java_cmd="java" + [ -n "${JAVA_HOME:-}" ] && [ -x "${JAVA_HOME}/bin/java" ] && java_cmd="${JAVA_HOME}/bin/java" + user_home="$("$java_cmd" -XshowSettings:properties -version 2>&1 \ + | sed -n 's/^ *user\.home = //p' | head -n 1)" + [ -n "$user_home" ] && repos="$repos ${user_home}/.m2/repository" + total=0 + for repo in $repos; do + [ -d "$repo" ] || continue + n="$(find "$repo" -name '*.lastUpdated' -type f 2>/dev/null | wc -l | tr -d ' ')" + if [ "${n:-0}" -gt 0 ]; then + find "$repo" -name '*.lastUpdated' -type f -delete 2>/dev/null || : + echo "retry.sh: cleared ${n} cached resolution failure(s) from ${repo}" >&2 + total=$((total + n)) + fi + done + # Said out loud either way: a purge that silently looked in the wrong place is + # how the case above went unnoticed. + if [ "$total" -eq 0 ]; then + echo "retry.sh: no cached resolution failures found under:${repos}" >&2 + fi +} + if [ "$#" -eq 0 ]; then echo "retry.sh: no command given" >&2 exit 2 @@ -106,6 +159,7 @@ while [ "$attempt" -le "$attempts" ]; do if [ "$attempt" -lt "$attempts" ]; then echo "retry.sh: attempt ${attempt}/${attempts} failed with status ${status};" \ "retrying in ${wait_seconds}s (possible transient Maven Central 403/429/5xx)" >&2 + purge_maven_negative_cache "$@" sleep "$wait_seconds" wait_seconds=$((wait_seconds * 4)) if [ "$wait_seconds" -gt "$max_delay" ]; then diff --git a/scripts/setup-workspace.sh b/scripts/setup-workspace.sh index 69b035f618c..158d38270ca 100755 --- a/scripts/setup-workspace.sh +++ b/scripts/setup-workspace.sh @@ -247,9 +247,30 @@ if [ -d "$CN1_BINARIES/.git" ]; then fi fi +# Retried like download_archive above. This clone was the one network step here that +# tried once: a runner that could not reach github.com for eight seconds -- "Failed to +# connect to github.com port 443" -- failed the whole job before anything was built. +# A failed clone can leave a partial directory behind, and git refuses to clone into +# a non-empty one, so each attempt starts from nothing. +clone_cn1_binaries() { + local delay + for delay in 0 15 60 180; do + if [ "$delay" -gt 0 ]; then + log "cn1-binaries clone failed; retrying in ${delay}s" + sleep "$delay" + fi + rm -rf "$CN1_BINARIES" + if git clone --depth=1 --filter=blob:none https://github.com/codenameone/cn1-binaries "$CN1_BINARIES"; then + return 0 + fi + done + log "could not clone cn1-binaries" >&2 + return 1 +} + if [ ! -d "$CN1_BINARIES/.git" ]; then log "Cloning cn1-binaries" - git clone --depth=1 --filter=blob:none https://github.com/codenameone/cn1-binaries "$CN1_BINARIES" + clone_cn1_binaries fi # Both builds below run with -T 1C, so several modules install into the local diff --git a/scripts/website/build.sh b/scripts/website/build.sh index 23d2868ad21..2a3ed1ef732 100755 --- a/scripts/website/build.sh +++ b/scripts/website/build.sh @@ -95,6 +95,56 @@ if [ "${WEBSITE_CN1_VERSION}" = "auto" ]; then fi fi +# Fetches the Maven wrapper jar a project's mvnw needs, with retries, before mvnw +# runs. The takari 0.5.6 wrapper downloads it ONCE with no retry, and a failed +# `wget -O` leaves an EMPTY jar behind that every later mvnw call trusts -- so one +# refused TLS handshake with Maven Central ("Unable to establish SSL connection") +# ended the website build as "Could not find or load main class +# org.apache.maven.wrapper.MavenWrapperMain". A jar that is not a complete zip is +# fetched again; a good one is left alone, so this costs nothing once it is there. +# Whether $1 is a COMPLETE zip archive. The first two bytes are not enough: a +# transfer cut off mid-body still starts with "PK", and that truncated jar was +# accepted as downloaded. unzip -t reads every entry and its CRC; python's +# zipfile does the same where unzip is missing. +is_complete_jar() { + [ -s "$1" ] || return 1 + if command -v unzip >/dev/null 2>&1; then + unzip -tq "$1" >/dev/null 2>&1 + elif command -v python3 >/dev/null 2>&1; then + python3 -m zipfile -t "$1" >/dev/null 2>&1 + else + echo "Neither unzip nor python3 is available to verify ${1}" >&2 + return 1 + fi +} + +ensure_maven_wrapper_jar() { + local dir="$1" props jar url delay + props="${dir}/.mvn/wrapper/maven-wrapper.properties" + jar="${dir}/.mvn/wrapper/maven-wrapper.jar" + [ -f "${props}" ] || return 0 + url="$(sed -n 's/^wrapperUrl=//p' "${props}" | tr -d '\r')" + # A wrapper with no wrapperUrl (the script-only kind) needs no jar. + [ -n "${url}" ] || return 0 + for delay in 0 15 60 180; do + if is_complete_jar "${jar}"; then + return 0 + fi + if [ "${delay}" -gt 0 ]; then + echo "Maven wrapper jar download failed; retrying in ${delay}s" >&2 + sleep "${delay}" + fi + rm -f "${jar}" + curl -fsSL --retry 3 --retry-delay 5 --retry-all-errors -o "${jar}" "${url}" || true + done + if is_complete_jar "${jar}"; then + return 0 + fi + rm -f "${jar}" + echo "Could not download the Maven wrapper jar from ${url}" >&2 + return 1 +} + build_javadocs_for_site() { if [ "${WEBSITE_INCLUDE_JAVADOCS}" != "true" ]; then return @@ -447,6 +497,7 @@ build_initializr_for_site() { echo "Building Initializr JavaScript bundle for website..." >&2 ( cd "${REPO_ROOT}/scripts/initializr" + ensure_maven_wrapper_jar "${PWD}" run_initializr_mvn() { if command -v xvfb-run >/dev/null 2>&1; then @@ -622,6 +673,7 @@ build_skindesigner_for_site() { echo "Building Skin Designer JavaScript bundle for website..." >&2 ( cd "${REPO_ROOT}/scripts/skindesigner" + ensure_maven_wrapper_jar "${PWD}" ./tools/sync-zipsupport-from-initializr.sh if [ "${WEBSITE_BOOTSTRAP_CN1_SNAPSHOTS}" = "true" ]; then activate_bootstrapped_java17 diff --git a/vm/backend/demo/oteltest/com/demo/OtelServer.java b/vm/backend/demo/oteltest/com/demo/OtelServer.java new file mode 100644 index 00000000000..1d88ecc03b4 --- /dev/null +++ b/vm/backend/demo/oteltest/com/demo/OtelServer.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import java.util.Map; + +import com.codename1.backend.Backend; +import com.codename1.backend.DataSource; +import com.codename1.backend.Database; +import com.codename1.backend.HttpServer; +import com.codename1.backend.Tracing; +import com.codename1.backend.Web; +import com.codename1.backend.WebSocket; +import com.codename1.backend.WebSocketSession; +import com.codename1.backend.orm.EntityManager; +import com.codename1.backend.otel.OtlpTracer; + +/** + * A traced server, for BackendOtelTest to drive on the PACKAGED runtime. + * + *

Built the way the generated entry point builds one -- the builder with a + * tracer -- and configured entirely from the environment, the way a deployment + * configures it: OTEL_EXPORTER_OTLP_ENDPOINT names the test's collector. + * + *

GET /work does the three things a span is made for in one request: a + * statement, and an outbound call that follows a redirect to a server the TEST + * runs (CN1_OTEL_DOWNSTREAM), which records the trace context that arrived. Not + * this server calling itself: on the packaged runtime an outbound call blocks the + * host thread it runs on, and the request it makes can be queued behind it on + * that same host. + * + *

/ws is a websocket whose onOpen runs a statement, so the test can see that a + * handshake is a span and that onOpen's work is its child. + */ +public class OtelServer { + public static void main(String[] args) throws Exception { + final String downstream = System.getenv("CN1_OTEL_DOWNSTREAM"); + final Database db = Database.open(":memory:"); + db.execute("CREATE TABLE pets (id INTEGER PRIMARY KEY, name TEXT)", null); + db.execute("INSERT INTO pets (id, name) VALUES (7, 'Rex')", null); + Backend.builder() + .tracing(new OtlpTracer("oteltest")) + .webSockets(new Backend.WebSocketEndpoints() { + public void register(HttpServer.WebSocketRegistry registry, + DataSource dataSource, EntityManager entities) { + registry.route("/ws", new WebSocket() { + public void onOpen(WebSocketSession session) throws Exception { + db.queryOne("SELECT name FROM pets WHERE id = ?", + new Object[] {Long.valueOf(7)}); + } + + public void onText(WebSocketSession session, String message) { + } + + public void onBinary(WebSocketSession session, byte[] message, + int offset, int length) { + } + }); + } + }) + .handler(new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) + throws Exception { + String path = request.getTarget(); + int query = path.indexOf('?'); + if(query >= 0) { + path = path.substring(0, query); + } + if("/boom".equals(path)) { + throw new IllegalStateException("boom"); + } + if(!"/work".equals(path)) { + return null; + } + Tracing.route("/work"); + Map row = db.queryOne("SELECT name FROM pets WHERE id = ?", + new Object[] {Long.valueOf(7)}); + // A GET with no headers of its own FOLLOWS a redirect, and + // the trace context this call carries must not change that. + Web.Result hop = Web.get(downstream + "/hop"); + return HttpServer.Response.text(200, row.get("name") + " " + + hop.getStatus() + " " + hop.getBodyAsString()); + } + }) + .run(); + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/Http2.java b/vm/backend/impl/javase/com/codename1/backend/Http2.java index ca24066f8e8..8729f92e576 100644 --- a/vm/backend/impl/javase/com/codename1/backend/Http2.java +++ b/vm/backend/impl/javase/com/codename1/backend/Http2.java @@ -144,6 +144,11 @@ public boolean isAlive() { return false; } + /** No session, so no stream ever closes. */ + public int[] closedStreams() { + return null; + } + /** Nothing is ever submitted here, so nothing is ever outstanding. */ public long pendingBodyBytes() { return 0; diff --git a/vm/backend/impl/javase/com/codename1/backend/Web.java b/vm/backend/impl/javase/com/codename1/backend/Web.java index fcacd4d75d2..c4f3780240b 100644 --- a/vm/backend/impl/javase/com/codename1/backend/Web.java +++ b/vm/backend/impl/javase/com/codename1/backend/Web.java @@ -228,6 +228,33 @@ public static Result request(String method, String url, List headers, byte[] bod // The request line's other field; see HeaderLines.requireMethod. HeaderLines.requireMethod(method); HeaderLines.validate(headers); + // The translated twin's span, the same way; see Tracing.startHttpClient. + Span span = Tracing.startHttpClient(method, url, headers); + int status = -1; + Throwable failure = null; + try { + Result result = perform(method, url, headers, body, + Tracing.propagationHeaders(span, headers)); + status = result.getStatus(); + return result; + } catch (IOException err) { + failure = err; + throw err; + } catch (RuntimeException err) { + failure = err; + throw err; + } finally { + Tracing.endHttpClient(span, status, failure); + } + } + + /** + * @param trace the trace-context lines, or null. Sent, but not counted as the + * caller's headers when deciding whether to follow a redirect -- + * the translated twin's rule, for the translated twin's reason. + */ + private static Result perform(String method, String url, List headers, byte[] body, + List trace) throws IOException { HttpURLConnection connection; try { connection = (HttpURLConnection)new URL(url).openConnection(); @@ -290,9 +317,17 @@ public static Result request(String method, String url, List headers, byte[] bod connection.setInstanceFollowRedirects((headers == null || headers.isEmpty()) && (body == null || body.length == 0) && safeMethod); connection.setRequestProperty("User-Agent", "codenameone-backend"); - if(headers != null) { - for(int iter = 0 ; iter < headers.size() ; iter++) { - String header = String.valueOf(headers.get(iter)); + List lines = headers; + if(trace != null) { + lines = new ArrayList(); + if(headers != null) { + lines.addAll(headers); + } + lines.addAll(trace); + } + if(lines != null) { + for(int iter = 0 ; iter < lines.size() ; iter++) { + String header = String.valueOf(lines.get(iter)); int colon = header.indexOf(':'); if(colon > 0) { // addRequestProperty, not set: the packaged client appends diff --git a/vm/backend/impl/parparvm/com/codename1/backend/Http2.java b/vm/backend/impl/parparvm/com/codename1/backend/Http2.java index 5f3c0ffc956..982eaf20f98 100644 --- a/vm/backend/impl/parparvm/com/codename1/backend/Http2.java +++ b/vm/backend/impl/parparvm/com/codename1/backend/Http2.java @@ -251,6 +251,19 @@ public byte[] drain() throws IOException { return drainImpl(session); } + /** + * The streams that have closed since the last call, as (stream id, HTTP/2 + * error code) pairs; null when none have. Code 0 is a stream whose response + * was sent in full; anything else is one reset before it was. A reset the + * PEER sent with NO_ERROR is reported as -1, not 0: it closes with code 0 too, + * and is no proof the response arrived. The server ends + * a request's span here, since a body the peer's flow-control window holds + * back is sent turns after it was submitted. + */ + public int[] closedStreams() { + return session == 0 ? null : closedStreamsImpl(session); + } + /** False once the session is finished and the connection can be closed. */ public boolean isAlive() { return wantsMoreImpl(session); @@ -324,6 +337,7 @@ private static void checkRange(byte[] buffer, int offset, int length) { private static native int pumpImpl(long session); private static native int pendingOutputImpl(long session); private static native byte[] drainImpl(long session); + private static native int[] closedStreamsImpl(long session); private static native int nextRequestImpl(long session); private static native String methodImpl(long session); private static native String pathImpl(long session); diff --git a/vm/backend/impl/parparvm/com/codename1/backend/Web.java b/vm/backend/impl/parparvm/com/codename1/backend/Web.java index ae1b6e4edab..6aa839e8e49 100644 --- a/vm/backend/impl/parparvm/com/codename1/backend/Web.java +++ b/vm/backend/impl/parparvm/com/codename1/backend/Web.java @@ -208,17 +208,64 @@ public static Result request(String method, String url, List headers, byte[] bod // The request line's other field; see HeaderLines.requireMethod. HeaderLines.requireMethod(method); HeaderLines.validate(headers); + // Null unless a tracer is installed; see Tracing.startHttpClient. + Span span = Tracing.startHttpClient(method, url, headers); + int status = -1; + Throwable failure = null; + try { + Result result = perform(method, url, headers, body, + Tracing.propagationHeaders(span, headers)); + status = result.getStatus(); + return result; + } catch (IOException err) { + failure = err; + throw err; + } catch (RuntimeException err) { + failure = err; + throw err; + } finally { + Tracing.endHttpClient(span, status, failure); + } + } + + /** + * @param trace the trace-context lines, or null. Sent like any other header, + * but NOT counted as the caller's: whether a redirect is followed + * freely depends on whether the caller handed over something that + * could leak (see CURLOPT_FOLLOWLOCATION in cn1_backend_web.c), + * and a trace id is not that. Counting it would have silently + * stopped every plain GET following redirects the moment tracing + * was turned on. + */ + private static Result perform(String method, String url, List headers, byte[] body, + List trace) throws IOException { StringBuilder joined = new StringBuilder(); + boolean callerHeaders = false; if(headers != null) { for(int iter = 0 ; iter < headers.size() ; iter++) { if(iter > 0) { joined.append('\n'); } - joined.append(String.valueOf(headers.get(iter))); + String line = String.valueOf(headers.get(iter)); + // What the native side would turn into a list entry: a line with + // something in it. An empty one never reached libcurl before either. + if(line.length() > 0) { + callerHeaders = true; + } + joined.append(line); + } + } + if(trace != null) { + for(int iter = 0 ; iter < trace.size() ; iter++) { + if(joined.length() > 0) { + joined.append('\n'); + } + joined.append(String.valueOf(trace.get(iter))); } } initialiseCurlOnce(); - long handle = performImpl(method, url, HeaderLines.narrowed(joined.toString()), body); + long handle = performImpl(method, url, HeaderLines.narrowed(joined.toString()), body, + callerHeaders); if(handle == 0) { // REDACTED, because this message goes wherever the caller logs it and // the URL may be a presigned one whose signature is the credential. @@ -332,7 +379,7 @@ private static synchronized void initialiseCurlOnce() throws IOException { private static native int globalInitImpl(); private static native long performImpl(String method, String url, byte[] headerLines, - byte[] body); + byte[] body, boolean callerHeaders); private static native String headersImpl(long handle); private static native int statusImpl(long handle); private static native String errorImpl(long handle); diff --git a/vm/backend/native/cn1_backend_http2.c b/vm/backend/native/cn1_backend_http2.c index 3e904cd7eb5..4d4aa5ed786 100644 --- a/vm/backend/native/cn1_backend_http2.c +++ b/vm/backend/native/cn1_backend_http2.c @@ -127,6 +127,19 @@ typedef struct { size_t outCapacity; /* Response bodies still being written, one per stream. See CN1H2Body. */ struct CN1H2Body* bodies; + /* Streams closed since Java last asked, as (id, error code) pairs. A server + span ends when its stream closes -- a response is fully sent only then, + which for a body held back by the peer's flow-control window is turns + after it was submitted -- and a nonzero code is a stream reset before it + was. Grown by doubling; drained by closedStreamsImpl every flush. */ + JAVA_INT* closed; + int closedPairs; + int closedCapacity; + /* The stream a RST_STREAM was just received for. nghttp2 delivers the frame + and then closes the stream within the same receive, and a reset with + NO_ERROR closes with error code 0 -- exactly like a response sent in full. + This is how the close tells the two apart. */ + int32_t resetStream; } CN1H2Session; /* @@ -824,6 +837,10 @@ static int cn1H2OnFrameRecv(nghttp2_session* session, const nghttp2_frame* frame CN1H2Request* r; int endStream = (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) != 0; (void)session; + if(frame->hd.type == NGHTTP2_RST_STREAM) { + s->resetStream = frame->hd.stream_id; + return 0; + } if(frame->hd.type != NGHTTP2_HEADERS && frame->hd.type != NGHTTP2_DATA) { return 0; } @@ -854,7 +871,32 @@ static int cn1H2OnStreamClose(nghttp2_session* session, int32_t streamId, CN1H2Session* s = (CN1H2Session*)userData; CN1H2Request* r = cn1H2FindOpen(s, streamId); (void)session; - (void)errorCode; + if(s->closedPairs == s->closedCapacity) { + int grown = s->closedCapacity == 0 ? 16 : s->closedCapacity * 2; + JAVA_INT* bigger = (JAVA_INT*)realloc(s->closed, (size_t)grown * 2 * sizeof(JAVA_INT)); + /* Out of memory: the entry is lost, and Java then ends that span as an + unwritten response when the connection goes -- a wrong status on one + span, never a crash or a leak. */ + if(bigger != NULL) { + s->closed = bigger; + s->closedCapacity = grown; + } + } + if(s->closedPairs < s->closedCapacity) { + JAVA_INT code = (JAVA_INT)errorCode; + if(streamId == s->resetStream) { + /* The peer reset it. With NO_ERROR that is still a stream the response + may not have finished on -- a client cancelling a download -- so it + is reported as a failure, never as delivery. */ + s->resetStream = 0; + if(code == 0) { + code = -1; + } + } + s->closed[s->closedPairs * 2] = (JAVA_INT)streamId; + s->closed[s->closedPairs * 2 + 1] = code; + s->closedPairs++; + } if(r != NULL) { /* Reset before it completed: drop it rather than leak the stream state. */ cn1H2Unlink(&s->open, r); @@ -1015,6 +1057,22 @@ JAVA_VOID com_codename1_backend_Http2_setMaxFileBodiesImpl___int(CODENAME_ONE_TH } /* Takes everything nghttp2 wants written, and empties the buffer. */ +/* The streams closed since the last call, as (id, error code) pairs, and forgets + them; null when there are none, so a turn with nothing closed allocates nothing. */ +JAVA_OBJECT com_codename1_backend_Http2_closedStreamsImpl___long_R_int_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + JAVA_OBJECT arr; + if(s == NULL || s->closedPairs == 0) { + return JAVA_NULL; + } + arr = allocArray(threadStateData, s->closedPairs * 2, &class_array1__JAVA_INT, + sizeof(JAVA_ARRAY_INT), 1); + memcpy((JAVA_ARRAY_INT*)((JAVA_ARRAY)arr)->data, s->closed, + (size_t)s->closedPairs * 2 * sizeof(JAVA_ARRAY_INT)); + s->closedPairs = 0; + return arr; +} + JAVA_OBJECT com_codename1_backend_Http2_drainImpl___long_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; JAVA_OBJECT arr; @@ -1551,6 +1609,7 @@ JAVA_VOID com_codename1_backend_Http2_destroyImpl___long(CODENAME_ONE_THREAD_STA } cn1H2FreeRequest(s->current); free(s->out); + free(s->closed); while(s->bodies != NULL) { CN1H2Body* next = s->bodies->next; cn1H2FreeBody(s->bodies); diff --git a/vm/backend/native/cn1_backend_web.c b/vm/backend/native/cn1_backend_web.c index 3e1f08716a6..c80436d4a19 100644 --- a/vm/backend/native/cn1_backend_web.c +++ b/vm/backend/native/cn1_backend_web.c @@ -209,9 +209,15 @@ JAVA_INT com_codename1_backend_Web_globalInitImpl___R_int(CODENAME_ONE_THREAD_ST /* * headerLines is one string with '\n' between headers, because passing a * String[] would mean walking a Java array from C for no benefit. + * + * callerHeaders says whether any of those lines came from the CALLER. The rest + * are trace context the runtime adds itself (see Web.perform), and the redirect + * rule below is about what the caller handed over, so it asks this rather than + * whether the list is empty. */ -JAVA_LONG com_codename1_backend_Web_performImpl___java_lang_String_java_lang_String_byte_1ARRAY_byte_1ARRAY_R_long( - CODENAME_ONE_THREAD_STATE, JAVA_OBJECT method, JAVA_OBJECT url, JAVA_OBJECT headerLines, JAVA_OBJECT body) { +JAVA_LONG com_codename1_backend_Web_performImpl___java_lang_String_java_lang_String_byte_1ARRAY_byte_1ARRAY_boolean_R_long( + CODENAME_ONE_THREAD_STATE, JAVA_OBJECT method, JAVA_OBJECT url, JAVA_OBJECT headerLines, JAVA_OBJECT body, + JAVA_BOOLEAN callerHeaders) { CURL* curl; CURLcode rc; struct curl_slist* headers = NULL; @@ -433,7 +439,7 @@ JAVA_LONG com_codename1_backend_Web_performImpl___java_lang_String_java_lang_Str wrong in; a NULL method is libcurl's own default GET. */ safeMethod = methodCopy == NULL || strcmp(methodCopy, "GET") == 0 || strcmp(methodCopy, "HEAD") == 0; - if(headers == NULL && bodyLength == 0 && safeMethod) { + if(!callerHeaders && bodyLength == 0 && safeMethod) { curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); } else { /* This #ifdef may never fire, for the same reason the PATH_AS_IS one diff --git a/vm/backend/src/com/codename1/backend/Backend.java b/vm/backend/src/com/codename1/backend/Backend.java index cdda046269e..c75192c6407 100644 --- a/vm/backend/src/com/codename1/backend/Backend.java +++ b/vm/backend/src/com/codename1/backend/Backend.java @@ -77,14 +77,22 @@ public final class Backend { private final EntityManager entities; private final Config config; private final int shutdownMillis; + /** + * The tracer this server installed, or null. The INSTANCE, not a flag: the + * global slot may hold another tracer by the time this server stops -- a + * second server in the same process, or one the application installed -- and + * stopping this one must not shut that down. + */ + private final Tracer ownTracer; private Backend(HttpServer server, DataSource dataSource, EntityManager entities, - Config config, int shutdownMillis) { + Config config, int shutdownMillis, Tracer ownTracer) { this.server = server; this.dataSource = dataSource; this.entities = entities; this.config = config; this.shutdownMillis = shutdownMillis; + this.ownTracer = ownTracer; } /** A builder whose defaults come from the configuration this process sees. */ @@ -134,6 +142,12 @@ public void stop() { if(dataSource != null) { dataSource.close(); } + // LAST, so the spans of the requests the drain let finish are exported + // rather than lost with the process -- and, when a request handler is the + // caller, after THAT request's span has ended, which is after this returns. + if(ownTracer != null) { + Tracing.shutdownAfterServing(ownTracer, shutdownMillis); + } } /** @@ -221,6 +235,7 @@ public static final class Builder { private boolean createTablesGiven; private boolean handlersNeedADatabase; private boolean quiet; + private Tracer tracer; Builder(Config config) { this.config = config; @@ -405,6 +420,17 @@ public Builder requiresDataSource() { return this; } + /** + * Traces every request with this tracer, once {@link Tracer#open} has read + * the configuration and agreed to. The build calls this from the entry + * point it generates for a project that enables OpenTelemetry, which is + * why nothing else refers to a tracer implementation. + */ + public Builder tracing(Tracer tracer) { + this.tracer = tracer; + return this; + } + /** Suppresses the line this prints when the server comes up. */ public Builder quiet() { this.quiet = true; @@ -419,9 +445,33 @@ public Backend start() throws Exception { if(config == null) { config = Config.load(); } + // BEFORE the database, so the statements start-up runs -- the ORM's + // CREATE TABLE -- are traced like any other, and before anything that + // could fail, so a refused configuration is refused up front. + boolean tracing = tracer != null && tracer.open(config); + // Installed without stopping whatever tracer was there before -- another + // server's, or one the program installed -- which is retired only once + // this start-up commits, and put back if it does not. + Tracing.Swap claim = tracing ? Tracing.swap(tracer) : null; + Backend started; + try { + started = startTraced(tracing); + } catch (Exception err) { + if(tracing) { + Tracing.rollBack(claim); + } + throw err; + } + if(tracing) { + Tracing.commit(claim); + } + return started; + } + + private Backend startTraced(boolean tracing) throws Exception { DataSource pool = openDataSource(); try { - return startWith(pool); + return startWith(pool, tracing); } catch (Exception err) { // EVERY failure after the pool is open, not just the bind. A // controller constructor that rejects its configuration, a @@ -444,9 +494,16 @@ public Backend start() throws Exception { } /** {@link #start} once the database, if any, is open. */ - private Backend startWith(DataSource pool) throws Exception { + private Backend startWith(DataSource pool, boolean tracing) throws Exception { EntityManager manager = openEntityManager(pool); - List routers = new ArrayList(handlers); + List routers = new ArrayList(); + HttpServer.Handler relay = tracing ? tracer.relay() : null; + if(relay != null) { + // FIRST: it answers one exact path, and a catch-all handler + // added before it would otherwise take the app's exports. + routers.add(relay); + } + routers.addAll(handlers); if(factory != null) { HttpServer.Handler[] built = factory.create(pool, manager); if(built != null) { @@ -585,7 +642,8 @@ public void register(HttpServer.WebSocketRegistry registry) // The websocket routes went in through start() above, before the // listener began accepting -- registering them here instead left a // window in which a valid upgrade was answered as ordinary HTTP. - Backend backend = new Backend(server, pool, manager, config, drain); + Backend backend = new Backend(server, pool, manager, config, drain, + tracing ? tracer : null); if(!quiet) { announce(backend, listenPort, context != null); } diff --git a/vm/backend/src/com/codename1/backend/Config.java b/vm/backend/src/com/codename1/backend/Config.java index 72de0a3ebf6..ebb2ce623de 100644 --- a/vm/backend/src/com/codename1/backend/Config.java +++ b/vm/backend/src/com/codename1/backend/Config.java @@ -141,6 +141,24 @@ public final class Config { private static final String[] WELL_KNOWN_ENVIRONMENT = { SERVER_PORT, "PORT", DATASOURCE_URL, "DATABASE_URL", + // The OpenTelemetry SDK's own variables, which every collector's + // documentation, every operator and every deployment template already + // uses. The keys are com.codename1.backend.otel.OtlpTracer's; they are + // spelled out here because this table is where the environment is mapped. + "cn1.otel.disabled", "OTEL_SDK_DISABLED", + "cn1.otel.endpoint", "OTEL_EXPORTER_OTLP_ENDPOINT", + "cn1.otel.traces.endpoint", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "cn1.otel.headers", "OTEL_EXPORTER_OTLP_HEADERS", + "cn1.otel.traces.headers", "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + "cn1.otel.protocol", "OTEL_EXPORTER_OTLP_PROTOCOL", + "cn1.otel.traces.protocol", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + "cn1.otel.service.name", "OTEL_SERVICE_NAME", + "cn1.otel.resource.attributes", "OTEL_RESOURCE_ATTRIBUTES", + "cn1.otel.sampler", "OTEL_TRACES_SAMPLER", + "cn1.otel.sampler.arg", "OTEL_TRACES_SAMPLER_ARG", + "cn1.otel.queue.size", "OTEL_BSP_MAX_QUEUE_SIZE", + "cn1.otel.batch.size", "OTEL_BSP_MAX_EXPORT_BATCH_SIZE", + "cn1.otel.export.delayMillis", "OTEL_BSP_SCHEDULE_DELAY", }; private final Properties profileFile; diff --git a/vm/backend/src/com/codename1/backend/Database.java b/vm/backend/src/com/codename1/backend/Database.java index dfdce5073dd..e79d089bbf7 100644 --- a/vm/backend/src/com/codename1/backend/Database.java +++ b/vm/backend/src/com/codename1/backend/Database.java @@ -254,6 +254,26 @@ public static Database of(Db db) throws IOException { */ public synchronized int execute(String sql, Object[] params) throws IOException { awaitTransactionOwner(); + // A span per statement when a tracer is installed; see Tracing.startDatabase. + Span span = Tracing.startDatabase(dialect.getName(), sql); + if(span == null) { + return executeUntraced(sql, params); + } + Throwable failure = null; + try { + return executeUntraced(sql, params); + } catch (IOException err) { + failure = err; + throw err; + } catch (RuntimeException err) { + failure = err; + throw err; + } finally { + Tracing.endDatabase(span, failure); + } + } + + private int executeUntraced(String sql, Object[] params) throws IOException { params = portableParameters(params); String rendered = bind(sql, params); if(sqlite != null) { @@ -268,6 +288,29 @@ public synchronized int execute(String sql, Object[] params) throws IOException /** Runs a query and returns every row as a column-name to value map. */ public synchronized List query(String sql, Object[] params) throws IOException { awaitTransactionOwner(); + Span span = Tracing.startDatabase(dialect.getName(), sql); + if(span == null) { + return queryUntraced(sql, params); + } + Throwable failure = null; + try { + List rows = queryUntraced(sql, params); + // Through the guarded hook: a tracer that throws here must not turn a + // query that succeeded into a failure, and discard its rows. + Tracing.setAttribute(span, "db.response.returned_rows", rows.size()); + return rows; + } catch (IOException err) { + failure = err; + throw err; + } catch (RuntimeException err) { + failure = err; + throw err; + } finally { + Tracing.endDatabase(span, failure); + } + } + + private List queryUntraced(String sql, Object[] params) throws IOException { params = portableParameters(params); String rendered = bind(sql, params); if(sqlite != null) { @@ -332,6 +375,29 @@ public synchronized Map queryOne(String sql, Object[] params) throws IOException public synchronized long insert(String sql, Object[] params, String idColumn) throws IOException { awaitTransactionOwner(); + // One span for the insert, whatever it runs to learn its key: the + // statements it issues through execute and query find this one current + // and add none of their own. + Span span = Tracing.startDatabase(dialect.getName(), sql); + if(span == null) { + return insertUntraced(sql, params, idColumn); + } + Throwable failure = null; + try { + return insertUntraced(sql, params, idColumn); + } catch (IOException err) { + failure = err; + throw err; + } catch (RuntimeException err) { + failure = err; + throw err; + } finally { + Tracing.endDatabase(span, failure); + } + } + + private long insertUntraced(String sql, Object[] params, String idColumn) + throws IOException { if(idColumn == null || idColumn.length() == 0) { throw new IOException("insert needs the name of the generated key column"); } diff --git a/vm/backend/src/com/codename1/backend/HttpServer.java b/vm/backend/src/com/codename1/backend/HttpServer.java index ad032aabd9d..ec02a04d38a 100644 --- a/vm/backend/src/com/codename1/backend/HttpServer.java +++ b/vm/backend/src/com/codename1/backend/HttpServer.java @@ -1966,6 +1966,9 @@ public Map getMetrics() { // enormous, so a leak surfaces hours later as a server that cannot // accept sockets, with nothing pointing at the cause. out.put("openStaticFiles", new Integer(StaticFiles.openFileCount())); + // Only when tracing is on, so a server that does not trace reports + // exactly what it always has. + Tracing.metrics(out); return out; } @@ -2206,10 +2209,14 @@ public void stop(int drainMillis) { synchronized(http2Sessions) { java.util.Iterator it = new java.util.ArrayList(http2Sessions.keySet()).iterator(); while(it.hasNext()) { - Object h2 = http2Sessions.remove(it.next()); + Object key = it.next(); + Object h2 = http2Sessions.remove(key); if(h2 != null) { ((Http2)h2).close(); } + if(key instanceof Integer) { + abandonHttp2Spans(((Integer)key).intValue()); + } } } if(tls != null) { @@ -3318,6 +3325,7 @@ private void drop(int fd) { if(h2 != null) { ((Http2)h2).close(); } + abandonHttp2Spans(fd); Object session = sessions.remove(new Integer(fd)); if(session != null) { Tls.closeSession(((Long)session).longValue()); @@ -3352,6 +3360,12 @@ private boolean isUpgradeRequest(Request request) { private WebSocket routeWebSocket(Request request, String path) throws Exception { Object exact = webSocketRoutes.get(path); if(exact != null) { + // Names the handshake span as the generated HTTP routers name theirs, + // or every endpoint's handshake is one operation called "GET". The + // registered path is literal (refused with a '%' or a '?') and matched + // exactly, so it IS the route template. A fallback router names its + // own, as any hand-written router does. + Tracing.route(path); return (WebSocket)exact; } WebSocketHandler router = webSocketRouter; @@ -3366,7 +3380,7 @@ private WebSocket routeWebSocket(Request request, String path) throws Exception * means the request was refused with a status and the connection is still an * ordinary HTTP one. */ - private boolean tryUpgrade(Conn conn, int fd, long session, Request request) { + private boolean tryUpgrade(Conn conn, int fd, long session, Request request, Span span) { // THE CANONICAL PATH, which is what pathIs compares for every HTTP route. // Taking the raw target substring instead meant `/ch%61t` missed the // websocket route for `/chat` and fell through to the catch-all router or @@ -3479,6 +3493,9 @@ private boolean tryUpgrade(Conn conn, int fd, long session, Request request) { writeHandshakeResponse(conn, WebSocketHandshake.accept(key), subprotocol); } catch (IOException err) { trace("fd=" + fd + " handshake write failed: " + err); + // Handled here, not by the caller: this path reports the connection + // as taken, so the caller will not end the handshake's span. + Tracing.endServer(span, -1, null); drop(fd); return true; } @@ -3490,6 +3507,7 @@ private boolean tryUpgrade(Conn conn, int fd, long session, Request request) { armWebSocketDeadline(fd); applyWebSocketReadTimeout(fd); + Exception onOpenError = null; try { endpoint.onOpen(socket); } catch (Exception err) { @@ -3501,7 +3519,13 @@ private boolean tryUpgrade(Conn conn, int fd, long session, Request request) { // is the same failure and gets the same answer. reportWebSocketError(socket, err); socket.failOnOpen(); + onOpenError = err; } + // The handshake's span ends HERE, with the 101 that was sent: it covered + // routing, the subprotocol choice and onOpen -- so work onOpen starts is + // its child -- and not the session, which can last for hours. Messages + // after this are not spans of their own. + Tracing.endServer(span, 101, onOpenError); runWebSocket(fd, socket); return true; } @@ -3908,6 +3932,7 @@ private void writeUpgradeRequired(Conn conn) { conn.put("Content-Length: 0\r\n"); conn.put("Connection: close\r\n\r\n"); writeTo(conn.fd, conn.session, conn.out, 0, conn.outLength); + conn.writtenStatus = 426; } catch (IOException ignored) { // The peer is already gone; there is nothing better to do here. } @@ -3980,6 +4005,12 @@ private static final class ProtocolException extends IOException { * it is what pipelining is, and what a proxy does when it coalesces. */ private final class Conn { + /** + * The status of the last refusal or handshake written on this connection, + * -1 when none was. A websocket handshake's span reads it: tryUpgrade + * answers through several writers, and each records what it sent. + */ + int writtenStatus = -1; final int fd; final long session; byte[] buffer = new byte[0]; @@ -4828,9 +4859,25 @@ private void serveOneRelease(int fd) { // it was never told carried a websocket. After wantsKeepAlive, so // nothing in the ordinary flow above moves. if(isUpgradeRequest(request)) { - if(tryUpgrade(conn, fd, session, request)) { + // The handshake is a request like any other and gets a span like + // any other. Started before tryUpgrade, so the websocket router, + // getSubprotocols and onOpen run inside it; tryUpgrade ends it once + // the upgrade is done, and a refusal ends here with the status it + // wrote. Returning before this point left every handshake, and + // everything onOpen called out to, untraced. + Span handshake = Tracing.startServer(request, tls != null); + conn.writtenStatus = -1; + boolean upgraded; + try { + upgraded = tryUpgrade(conn, fd, session, request, handshake); + } catch (RuntimeException err) { + Tracing.endServer(handshake, -1, err); + throw err; + } + if(upgraded) { return; // this connection is no longer HTTP } + Tracing.endServer(handshake, conn.writtenStatus, null); // REFUSED, AND THE REFUSAL SAID `Connection: close`. Carrying on // to parse whatever the client pipelined behind the handshake // would execute a second request the server has already promised @@ -4846,6 +4893,14 @@ private void serveOneRelease(int fd) { // the whole of serveOne: that is the CONNECTION, which outlives this. inFlightRequests.incrementAndGet(); SERVING_FD.set(new Integer(fd)); + // Null unless a tracer is installed. Started HERE, before the handler, + // and read from the request now: the Request is reused by the next + // one on this connection. Ended after the write, in the finally below, + // so the span covers the response reaching the socket and a write that + // fails is recorded as the failure it is. + Span span = Tracing.startServer(request, tls != null); + int sentStatus = -1; + Exception handlerError = null; try { try { response = handler.handle(request); @@ -4854,10 +4909,14 @@ private void serveOneRelease(int fd) { } } catch (Exception err) { System.err.println("handler failed: " + err); + handlerError = err; response = Response.text(500, "internal error"); } + // Read before the write: writing releases what the Response held. + int status = response.status; try { writeResponse(conn, fd, session, response, keepAlive, headOnly); + sentStatus = status; if(conn.stripe >= 0) { servedStripes[conn.stripe]++; // single writer: this host } else { @@ -4871,6 +4930,9 @@ private void serveOneRelease(int fd) { } finally { inFlightRequests.decrementAndGet(); SERVING_FD.set(null); + if(span != null) { + Tracing.endServer(span, sentStatus, handlerError); + } } if(!keepAlive) { drop(fd); @@ -5204,6 +5266,23 @@ private void serveHttp2(int fd, long session, byte[] pending, int pendingLength) inFlightRequests.incrementAndGet(); SERVING_FD.set(new Integer(fd)); SERVING_H2.set(Boolean.TRUE); + // The HTTP/1 path's span, for the same reasons. It stops being this + // thread's current span in the finally that closes this stream's + // request, and ENDS when its stream closes (see http2Spans). + // server.address is set here as for HTTP/1: :authority was copied + // into these headers as "host" above, which is what startServer reads. + Span span = Tracing.startServer(request, tls != null); + Exception handlerError = null; + // -1 until the response has been SUBMITTED to the session, as on the + // HTTP/1 path: a respond() that throws is a response the peer never + // got, and must not be reported as the status the handler chose. + int submittedStatus = -1; + // The status the peer is actually sent when the session refuses the + // handler's response and a 503 goes in its place; -1 when it did not. + int fallbackStatus = -1; + // Whether the span has been handed to tracedStreams, which then + // owns ending it. + boolean spanRegistered = false; try { response = handler.handle(request); if(response == null) { @@ -5211,6 +5290,7 @@ private void serveHttp2(int fd, long session, byte[] pending, int pendingLength) } } catch (Exception err) { System.err.println("handler failed: " + err); + handlerError = err; response = Response.text(500, "internal error"); } try { @@ -5278,6 +5358,7 @@ private void serveHttp2(int fd, long session, byte[] pending, int pendingLength) asciiBytes("too many files in flight"))) { h2.respond(stream.getId(), 503, "text/plain", refusalHeaders(), null); } + fallbackStatus = 503; } else if(response.fileFd >= 0 && !noBody) { // Streamed frame by frame out of the descriptor. Reading the file // in first cost its whole size in the heap plus the same again in @@ -5297,6 +5378,7 @@ private void serveHttp2(int fd, long session, byte[] pending, int pendingLength) // happens in the same step that takes the descriptor. StaticFiles.closeFile(response.fileFd); h2.respond(stream.getId(), 503, "text/plain", refusalHeaders(), null); + fallbackStatus = 503; } } else { // A HEAD describes the representation it is not sending, and @@ -5355,15 +5437,32 @@ private void serveHttp2(int fd, long session, byte[] pending, int pendingLength) // is no room for bodies. An explanatory body here is the // one allocation that must not be attempted. h2.respond(stream.getId(), 503, "text/plain", refusalHeaders(), null); + fallbackStatus = 503; } else { queuedBodyBytes += bodyBytes; } } + // What the peer received: the 503 that replaced a refused response + // is what the client saw, and the span must say so -- recording the + // handler's 200 hid exactly the overload a trace is read to find. + submittedStatus = fallbackStatus > 0 ? fallbackStatus : response.status; requestsServed.incrementAndGet(); + if(span != null) { + // Registered NOW, before the flush just below can run: that + // flush may close this very stream, and settling consumes its + // one close notification. Registered after it -- in the finally + // -- the span missed that notification and stayed open until + // the whole connection went. + Tracing.leave(span); + tracedStreams(fd).put(new Integer(stream.getId()), + new Object[] {span, new Integer(submittedStatus), handlerError}); + spanRegistered = true; + } if(queuedBodyBytes > MAX_QUEUED_H2_BODY_BYTES || Http2.pendingBodyFiles() > MAX_OPEN_H2_FILES || Http2.pendingBodyBytesAll() > MAX_OPEN_H2_BODY_BYTES) { flushHttp2(fd, session, h2); + settleHttp2Spans(fd, h2); // What the flush could NOT write, not zero. nghttp2 pulls // from a submitted body only as the peer's flow-control // window allows, so a client that simply stops sending @@ -5395,9 +5494,25 @@ private void serveHttp2(int fd, long session, byte[] pending, int pendingLength) inFlightRequests.decrementAndGet(); SERVING_FD.set(null); SERVING_H2.set(null); + if(span != null && !spanRegistered) { + // Submitting threw before the span was registered above. Not + // current any more, so the next stream's span is not its + // child. + Tracing.leave(span); + if(submittedStatus < 0) { + // Nothing was submitted, so no stream close will ever + // report on it: it failed here. + Tracing.endServer(span, -1, handlerError); + } else { + tracedStreams(fd).put(new Integer(stream.getId()), + new Object[] {span, new Integer(submittedStatus), + handlerError}); + } + } } } flushHttp2(fd, session, h2); + settleHttp2Spans(fd, h2); if(!h2.isAlive()) { drop(fd); return; @@ -5412,6 +5527,79 @@ private void serveHttp2(int fd, long session, byte[] pending, int pendingLength) } } + /** + * Server spans of HTTP/2 responses submitted but not yet fully sent, per + * connection and then per stream id: each an Object[] {span, submitted status, + * handler error}. A response is sent in full only when its stream CLOSES -- + * nghttp2 pulls a body only as the peer's flow-control window allows, so a + * large one finishes turns after it was submitted -- and a stream the peer + * resets never is. Ending the span at submission, or at the first flush, + * reported a success for a response the peer never got. The HTTP/1 path keeps + * its span open through the write for the same reason. + */ + private final Map http2Spans = java.util.Collections.synchronizedMap(new java.util.HashMap()); + + private Map tracedStreams(int fd) { + Integer key = new Integer(fd); + synchronized(http2Spans) { + Map streams = (Map)http2Spans.get(key); + if(streams == null) { + streams = java.util.Collections.synchronizedMap(new java.util.HashMap()); + http2Spans.put(key, streams); + } + return streams; + } + } + + /** + * Ends the spans of the streams that closed, after a flush put their final + * frames on the socket: with the status sent when the stream closed cleanly, + * as the failed write HTTP/1 reports (-1) when it was reset. + */ + private void settleHttp2Spans(int fd, Http2 h2) { + int[] closed = h2.closedStreams(); + if(closed == null) { + return; + } + Map streams = (Map)http2Spans.get(new Integer(fd)); + if(streams == null) { + return; + } + for(int iter = 0 ; iter + 1 < closed.length ; iter += 2) { + Object entry = streams.remove(new Integer(closed[iter])); + if(entry instanceof Object[]) { + endHttp2Span((Object[])entry, closed[iter + 1] == 0); + } + } + } + + /** Every span still open on a connection that is going away: none was sent in full. */ + private void abandonHttp2Spans(int fd) { + Object streams = http2Spans.remove(new Integer(fd)); + if(!(streams instanceof Map)) { + return; + } + synchronized(streams) { + java.util.Iterator it = ((Map)streams).values().iterator(); + while(it.hasNext()) { + Object entry = it.next(); + if(entry instanceof Object[]) { + endHttp2Span((Object[])entry, false); + } + } + ((Map)streams).clear(); + } + } + + private static void endHttp2Span(Object[] entry, boolean sent) { + if(!(entry[0] instanceof Span)) { + return; + } + int status = sent && entry[1] instanceof Integer ? ((Integer)entry[1]).intValue() : -1; + Tracing.endServer((Span)entry[0], status, + entry[2] instanceof Exception ? (Exception)entry[2] : null); + } + /** The HTTP/2 connection preface, sent by a client that opens with h2. */ private static final byte[] HTTP2_PREFACE = prefaceBytes(); @@ -6082,8 +6270,17 @@ private void writeStatusOnly(Conn conn, int status, String message) { head.append("Date: ").append(currentHttpDate()).append("\r\n"); head.append("Content-Length: ").append(body.length).append("\r\n"); head.append("Connection: close\r\n\r\n"); - conn.write(head.toString().getBytes("UTF-8")); - conn.write(body); + // ONE write, as writeUpgradeRequired does. Written as head then body, a + // client that read the status line and closed -- leaving the rest + // unread, which makes the close a reset -- failed the second write, + // so writtenStatus was never set and the span of a 404 the client HAD + // received reported no status at all. Seen on a slow CI runner. + byte[] headBytes = head.toString().getBytes("UTF-8"); + byte[] whole = new byte[headBytes.length + body.length]; + System.arraycopy(headBytes, 0, whole, 0, headBytes.length); + System.arraycopy(body, 0, whole, headBytes.length, body.length); + conn.write(whole); + conn.writtenStatus = status; } catch (IOException err) { // The peer is already gone; there is nowhere to report this. } diff --git a/vm/backend/src/com/codename1/backend/LambdaRuntime.java b/vm/backend/src/com/codename1/backend/LambdaRuntime.java index 2b55eec0503..63db65bc2a9 100644 --- a/vm/backend/src/com/codename1/backend/LambdaRuntime.java +++ b/vm/backend/src/com/codename1/backend/LambdaRuntime.java @@ -40,6 +40,8 @@ public final class LambdaRuntime { private static final String API_VERSION = "/2018-06-01/runtime"; private static final String REQUEST_ID_HEADER = "Lambda-Runtime-Aws-Request-Id"; + /** The invocation's X-Ray trace, which is how the host passes the caller's trace on. */ + private static final String TRACE_ID_HEADER = "Lambda-Runtime-Trace-Id"; private LambdaRuntime() { } @@ -91,75 +93,124 @@ static boolean pumpOnce(Handler handler, String host, int port) { System.err.println("Invocation carried no " + REQUEST_ID_HEADER + "; cannot report a result"); return false; } + // The invocation's span when a tracer is installed, parented on the trace + // the host hands over. Ended as soon as the handler returns, and FLUSHED + // before the next poll: the host freezes this process while it waits for + // the next invocation, so a span still queued then is sent late or never. + Span span = Tracing.startLambda(next.getHeader(TRACE_ID_HEADER), requestId); + try { + return answer(handler, host, port, next, requestId, span); + } finally { + if(span != null) { + Tracing.flush(2000); + } + } + } + + private static boolean answer(Handler handler, String host, int port, Http.Response next, + String requestId, Span span) { String result; try { result = handler.handle(next.getBodyAsString(), requestId); } catch (Exception err) { + // The span stays open through the error report too, as it does through + // the result's delivery below: the report's latency is part of the + // invocation, and a report the host refuses or never receives leaves + // the invocation unresolved -- which the trace must say. + boolean reported = false; + try { + reported = reportError(host, port, requestId, err); + } finally { + Tracing.endLambda(span, err, reported ? null : unreported()); + } // The same rule the response path below takes, and for the same // reason: an invocation the host was never told about stays // outstanding until it times out, and polling for another one while // that is true just strands them one after the next. If the failure // could not even be reported, nothing this process says is reaching // the host, so it stops rather than collecting more. - if(!reportError(host, port, requestId, err)) { + if(!reported) { System.err.println("The runtime API is unreachable, so this runtime is " + "stopping rather than collecting invocations it cannot answer."); return false; } return true; } + // The span stays open through DELIVERY: a result the Runtime API refuses, + // or a POST that fails, is an invocation whose answer was lost, and the + // trace has to say so rather than report the handler's success. + Throwable delivery = null; + Throwable reporting = null; try { - byte[] payload = (result == null ? "null" : result).getBytes("UTF-8"); - // The status matters: the Runtime API REJECTS a result it will not take - // -- 413 for a payload over the response limit is the ordinary case -- - // and answers rather than throwing. Discarding it meant the handler's - // work was dropped and the loop went straight back to polling, with the - // caller left waiting for a reply that was never accepted and nothing - // anywhere saying why. - Http.Response posted = Http.post(host, port, - API_VERSION + "/invocation/" + requestId + "/response", payload); - if(posted == null || posted.getStatus() < 200 || posted.getStatus() >= 300) { - System.err.println("The Lambda runtime API refused the response for " - + requestId + " with status " - + (posted == null ? "none" : String.valueOf(posted.getStatus())) - + "; the result of " + payload.length + " byte(s) was not " - + "delivered. Reporting it as an error so the invocation " - + "does not simply hang."); - // And stop if even THAT could not be delivered. The result is - // already gone, so an unreported invocation stays outstanding - // until the host times it out while this loop takes the next - // one. Third branch with this rule; they are the three ways an - // invocation can end without the host being told. - if(!reportError(host, port, requestId, new java.io.IOException( - "the runtime API refused the response with status " - + (posted == null ? "none" : String.valueOf(posted.getStatus()))))) { + try { + byte[] payload = (result == null ? "null" : result).getBytes("UTF-8"); + // The status matters: the Runtime API REJECTS a result it will not take + // -- 413 for a payload over the response limit is the ordinary case -- + // and answers rather than throwing. Discarding it meant the handler's + // work was dropped and the loop went straight back to polling, with the + // caller left waiting for a reply that was never accepted and nothing + // anywhere saying why. + Http.Response posted = Http.post(host, port, + API_VERSION + "/invocation/" + requestId + "/response", payload); + if(posted == null || posted.getStatus() < 200 || posted.getStatus() >= 300) { + System.err.println("The Lambda runtime API refused the response for " + + requestId + " with status " + + (posted == null ? "none" : String.valueOf(posted.getStatus())) + + "; the result of " + payload.length + " byte(s) was not " + + "delivered. Reporting it as an error so the invocation " + + "does not simply hang."); + // And stop if even THAT could not be delivered. The result is + // already gone, so an unreported invocation stays outstanding + // until the host times it out while this loop takes the next + // one. Third branch with this rule; they are the three ways an + // invocation can end without the host being told. + java.io.IOException refused = new java.io.IOException( + "the runtime API refused the response with status " + + (posted == null ? "none" : String.valueOf(posted.getStatus()))); + delivery = refused; + if(!reportError(host, port, requestId, refused)) { + reporting = unreported(); + System.err.println("The runtime API is unreachable, so this runtime is " + + "stopping rather than collecting invocations it cannot answer."); + return false; + } + } + } catch (Exception err) { + delivery = err; + // The result is GONE -- it existed only in the request that just + // failed -- so this invocation has to be resolved here or it stays + // outstanding until the host times it out, while this loop cheerfully + // takes the next one. Reporting the failure is what lets the host + // fail it now instead. + System.err.println("Failed to post the response for " + requestId + ": " + err + + "; reporting it as an error so the invocation is resolved rather " + + "than left outstanding."); + if(!reportError(host, port, requestId, err)) { + reporting = unreported(); + // Not even the error reached the host, so nothing this process + // says is getting through. Stop polling: collecting further + // invocations only strands them the same way, and an exited + // runtime is something Lambda knows how to recover from. System.err.println("The runtime API is unreachable, so this runtime is " + "stopping rather than collecting invocations it cannot answer."); return false; } } - } catch (Exception err) { - // The result is GONE -- it existed only in the request that just - // failed -- so this invocation has to be resolved here or it stays - // outstanding until the host times it out, while this loop cheerfully - // takes the next one. Reporting the failure is what lets the host - // fail it now instead. - System.err.println("Failed to post the response for " + requestId + ": " + err - + "; reporting it as an error so the invocation is resolved rather " - + "than left outstanding."); - if(!reportError(host, port, requestId, err)) { - // Not even the error reached the host, so nothing this process - // says is getting through. Stop polling: collecting further - // invocations only strands them the same way, and an exited - // runtime is something Lambda knows how to recover from. - System.err.println("The runtime API is unreachable, so this runtime is " - + "stopping rather than collecting invocations it cannot answer."); - return false; - } + } finally { + Tracing.endLambda(span, delivery, reporting); } return true; } + /** + * Recorded on an invocation span when not even the error report reached the + * host: the invocation is left unresolved, which is why the runtime stops. + */ + private static java.io.IOException unreported() { + return new java.io.IOException( + "the runtime API did not accept the error report; the invocation is unresolved"); + } + /** @return whether the host accepted the report, so a caller can stop. */ private static boolean reportError(String host, int port, String requestId, Exception cause) { try { diff --git a/vm/backend/src/com/codename1/backend/Span.java b/vm/backend/src/com/codename1/backend/Span.java new file mode 100644 index 00000000000..a7d0d213b88 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/Span.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +/** + * One timed operation inside a distributed trace. + * + *

The server creates these itself -- one per request, one per outbound + * {@link Web} call and one per {@link Database} statement -- so an application + * normally only touches the current one, through {@link Tracing#current()}, to + * add an attribute. {@link Tracing#inSpan} wraps a block of its own work. + * + *

When tracing is not installed every method here is a no-op on a shared + * instance, so code that decorates the current span costs nothing in a server + * that does not export traces. + * + *

The kinds and status codes are the numbers the OTLP wire format uses, so a + * tracer can write them without translating. + */ +public abstract class Span { + /** An operation inside the process. */ + public static final int KIND_INTERNAL = 1; + /** Answering a request that arrived from outside. */ + public static final int KIND_SERVER = 2; + /** A request this process makes to something else. */ + public static final int KIND_CLIENT = 3; + + /** + * What was current when this span became current, restored when it ends. + * Owned by {@link Tracing}; a tracer never reads it. + */ + Span previous; + /** Whether {@link Tracing} made this span current and must restore on end. */ + boolean entered; + /** + * A tracer to shut down once this server span has ended, set when the + * request it describes stopped the server; see Tracing.shutdownAfterServing. + */ + Tracer shutdownOnEnd; + int shutdownOnEndMillis; + + /** For tracer implementations. */ + protected Span() { + } + + /** Adds or replaces a string attribute. A null value is ignored. */ + public abstract Span setAttribute(String key, String value); + + /** Adds or replaces an integer attribute. */ + public abstract Span setAttribute(String key, long value); + + /** Adds or replaces a floating point attribute. */ + public abstract Span setAttribute(String key, double value); + + /** Adds or replaces a flag attribute. */ + public abstract Span setAttribute(String key, boolean value); + + /** + * Records a failure as an "exception" event and marks the span as an error. + * The message is kept; the stack trace is not, because it can carry values a + * handler had in scope and it is by far the largest part of a span. + */ + public abstract Span recordException(Throwable error); + + /** Marks the span as failed, with a short description. */ + public abstract Span setError(String description); + + /** Renames the span. The router calls this once it knows the route template. */ + public abstract Span updateName(String name); + + /** The current name. */ + public abstract String getName(); + + /** One of the KIND constants. */ + public abstract int getKind(); + + /** + * Whether this span is being recorded. A span the sampler declined still + * carries a trace context and propagates it, so the NEXT service agrees not to + * record either -- but attributes set on it go nowhere, and a caller that + * would compute an expensive one can ask first. + */ + public abstract boolean isRecording(); + + /** + * This span as a W3C {@code traceparent} value, which is what an outbound + * request carries so the service it reaches joins the same trace. Null for + * the no-op span. + */ + public abstract String traceparent(); + + /** The W3C {@code tracestate} this trace carries, or null for none. */ + public abstract String tracestate(); + + /** + * Drops the span: it ends normally but is never exported. For traffic that is + * about tracing itself, which would otherwise report on every export. + */ + public abstract void discard(); + + /** Ends the span. Only the first call counts. */ + public abstract void end(); +} diff --git a/vm/backend/src/com/codename1/backend/Tracer.java b/vm/backend/src/com/codename1/backend/Tracer.java new file mode 100644 index 00000000000..62f45aa72dc --- /dev/null +++ b/vm/backend/src/com/codename1/backend/Tracer.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.util.Map; + +/** + * Where spans come from and where they go. + * + *

The server is instrumented against this interface and nothing else, so a + * binary that installs no tracer carries no tracing implementation: the + * translator keeps a class only when something reaches it, and the one + * implementation, {@code com.codename1.backend.otel.OtlpTracer}, is reached only + * from the entry point the build generates for a project that asks for it (see + * {@code @OpenTelemetry}). + */ +public interface Tracer { + /** + * Reads the settings and starts whatever exports spans. Called once, by + * {@link Backend.Builder#start} or by the application before + * {@link Tracing#install}. + * + * @return false when the configuration turned tracing off, in which case the + * tracer is not installed and nothing was started + */ + boolean open(Config config) throws IOException; + + /** + * A new span. Never null: a span the sampler declines is a non-recording one + * that still carries its trace context. + * + * @param parent the local parent, or null + * @param traceparent a remote parent's {@code traceparent} header, used when + * {@code parent} is null; null or malformed starts a new trace + * @param tracestate the remote parent's {@code tracestate}, or null + */ + Span startSpan(String name, int kind, Span parent, String traceparent, String tracestate); + + /** + * Blocks until what has ended so far is exported, or the time runs out. A + * Lambda host freezes the process between invocations, so a background + * exporter there only runs when something waits for it. + */ + void flush(int timeoutMillis); + + /** Flushes, then stops the exporter. */ + void shutdown(int timeoutMillis); + + /** + * The handler that relays client spans to the collector, or null when this + * deployment does not offer one. The builder puts it ahead of the routers. + */ + HttpServer.Handler relay(); + + /** Adds this tracer's counters to a metrics snapshot. */ + void metrics(Map out); +} diff --git a/vm/backend/src/com/codename1/backend/Tracing.java b/vm/backend/src/com/codename1/backend/Tracing.java new file mode 100644 index 00000000000..3994a7c5074 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/Tracing.java @@ -0,0 +1,1067 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Distributed tracing for the server, and the hooks it is instrumented through. + * + *

Nothing here does anything until a {@link Tracer} is installed. The build + * installs one when a project asks for it -- {@code @OpenTelemetry} on any class, + * or {@code cn1.otel.enabled=true} in application.properties -- and from then on + * every request, outbound {@link Web} call and {@link Database} statement is a + * span, W3C trace context travels on every outbound request, and an incoming + * {@code traceparent} makes the request part of the caller's trace. No code in + * the application changes for any of that. + * + *

What an application may still want is here: {@link #current()} to decorate + * the request's span, and {@link #inSpan} to time a block of its own work. + * + *

THE CURRENT SPAN IS PER THREAD, and that is per REQUEST here: a virtual + * thread serves one request at a time and ThreadLocal is per virtual thread (see + * the measurement recorded beside HttpServer.SERVING_FD). The server sets it + * before the handler runs and clears it in the same finally that ends the + * request, because the next request on a kept-alive connection runs on the same + * thread and must not inherit it. + * + *

Every hook is guarded: a tracer that throws is reported once and the + * request goes on untraced, because a monitoring fault must never become an + * outage. + */ +public final class Tracing { + /** The installed tracer, or null. Volatile: installed once, read by every host. */ + private static volatile Tracer tracer; + private static final ThreadLocal CURRENT = new ThreadLocal(); + private static final ThreadLocal SUPPRESSED = new ThreadLocal(); + private static volatile boolean reportedFailure; + private static final Span NOOP = new NoopSpan(); + + /** The W3C header names, lower case as HTTP/2 requires. */ + static final String TRACEPARENT = "traceparent"; + static final String TRACESTATE = "tracestate"; + + private Tracing() { + } + + /** What {@link #inSpan} runs. */ + public interface Work { + Object run(Span span) throws Exception; + } + + /** How long a replaced tracer gets to export what it already holds. */ + static final int REPLACED_SHUTDOWN_MILLIS = 2000; + + /** + * Installs the tracer every hook reports to, replacing any earlier one. Pass + * null to turn tracing off. The builder does this itself; a program that + * starts {@link HttpServer} or {@link LambdaRuntime} directly calls it after + * {@link Tracer#open}. + * + *

A tracer this replaces is shut down: its spans so far are exported, for + * up to {@link #REPLACED_SHUTDOWN_MILLIS}, and its exporter stops. Nothing + * else holds it once it is out of this slot, so leaving it running leaked its + * export thread and queues on every reconfiguration. + */ + public static void install(Tracer installed) { + commit(swap(installed)); + } + + /** + * A start-up's claim on the slot, from {@link #swap} until {@link #commit} or + * {@link #rollBack}: what it installed, and what that displaced. + * + *

{@code previous} can CHANGE while the claim is open. Two start-ups that + * overlap chain: A swaps X out for F, B swaps F out for G. If A then fails, F + * is gone but X is not coming back yet -- B is still starting over it -- so + * A's rollback hands X to B's claim. B's rollback then restores X and its + * commit retires X, and each failed tracer is stopped exactly once. Without + * the chain, A retired X and B's rollback restored A's failed F. + */ + static final class Swap { + final Tracer installed; + Tracer previous; + + Swap(Tracer installed, Tracer previous) { + this.installed = installed; + this.previous = previous; + } + } + + /** Claims made by swap() and not yet committed or rolled back. Under LIFECYCLE. */ + private static final List PENDING = new ArrayList(); + + /** + * Installs {@code installed} WITHOUT stopping the tracer it replaces. For a + * start-up that can still fail: it traces its own start-up with the new + * tracer, then {@link #commit}s if it completed or {@link #rollBack}s if it + * did not. Shutting the old one down up front left a failed second server's + * process with no tracer at all, while the first server kept running untraced. + */ + static Swap swap(Tracer installed) { + synchronized(LIFECYCLE) { + Swap claim = new Swap(installed, tracer); + tracer = installed; + PENDING.add(claim); + return claim; + } + } + + /** + * Guards every read-and-replace of the slot, and the chain of open claims. + * Two servers starting at once each read the same previous tracer before + * either write landed, so each retired that one, and the tracer the FIRST + * installed -- overwritten by the second -- was never shut down. The server + * is multi-threaded here; the lock is never held across a shutdown, which + * can block for seconds. + */ + private static final Object LIFECYCLE = new Object(); + + /** A start-up completed: what its tracer displaced is stopped. */ + static void commit(Swap claim) { + Tracer displaced; + synchronized(LIFECYCLE) { + PENDING.remove(claim); + displaced = claim.previous; + } + retire(displaced, claim.installed); + } + + /** Stops {@code previous}, replaced by {@code installed}, exporting what it held. */ + static void retire(Tracer previous, Tracer installed) { + if(previous != null && previous != installed) { + try { + previous.shutdown(REPLACED_SHUTDOWN_MILLIS); + } catch (RuntimeException err) { + failed(err); + } + } + } + + /** + * Undoes a {@link #swap}. Three cases, decided under the lock: + *

    + *
  • its tracer is still installed: what it displaced goes back, and the + * failed tracer stops;
    • + *
  • a start-up that is still open swapped over it: that claim inherits + * what this one displaced (see {@link Swap}), and the failed tracer + * stops;
    • + *
  • an install or a commit replaced it, and retired it doing so: what it + * displaced is not coming back, and is retired here, since nothing else + * holds it.
    • + *
+ */ + static void rollBack(Swap claim) { + Tracer stopFailed = null; + Tracer orphan = null; + synchronized(LIFECYCLE) { + PENDING.remove(claim); + if(tracer == claim.installed) { + tracer = claim.previous; + stopFailed = claim.installed; + } else { + Swap later = null; + for(int iter = 0 ; iter < PENDING.size() ; iter++) { + Swap open = (Swap)PENDING.get(iter); + if(open.previous == claim.installed) { + later = open; + break; + } + } + if(later != null) { + later.previous = claim.previous; + stopFailed = claim.installed; + } else { + orphan = claim.previous; + } + } + } + if(stopFailed != null && stopFailed != claim.previous) { + try { + // The same window a replaced tracer gets, not none: a start-up that + // failed after its database initialisation holds exactly the spans + // that explain the failure, and shutdown(0) dropped them unsent. + stopFailed.shutdown(REPLACED_SHUTDOWN_MILLIS); + } catch (RuntimeException err) { + failed(err); + } + } + if(orphan != null) { + retire(orphan, claim.installed); + } + } + + /** The installed tracer, or null. */ + public static Tracer getTracer() { + return tracer; + } + + /** Whether a tracer is installed. */ + public static boolean isEnabled() { + return tracer != null; + } + + /** + * The span of the work this thread is doing: the request's, inside a handler. + * Never null -- with no tracer, or outside a request, it is a no-op span. + */ + public static Span current() { + Span span = currentOrNull(); + return span == null ? NOOP : span; + } + + /** + * The current trace as a W3C {@code traceparent} value, for a transport the + * server does not instrument itself -- a message queue, a raw socket. Null + * when there is no trace. + */ + public static String currentTraceparent() { + Span span = currentOrNull(); + return span == null ? null : span.traceparent(); + } + + /** + * Runs {@code work} inside a new span that is a child of the current one, and + * is current itself while it runs. An exception is recorded on the span and + * rethrown. + */ + public static Object inSpan(String name, Work work) throws Exception { + Span span = begin(name, Span.KIND_INTERNAL, null, null); + if(span == null) { + return work.run(NOOP); + } + try { + return work.run(span); + } catch (Exception err) { + guardedException(span, err); + throw err; + } finally { + finish(span); + } + } + + /** + * A new child of the current span that is NOT made current, for work whose + * start and end are in different places. The caller must end it. + */ + public static Span startSpan(String name) { + Tracer t = tracer; + if(t == null || isSuppressed()) { + return NOOP; + } + try { + Span span = t.startSpan(name, Span.KIND_INTERNAL, currentOrNull(), null, null); + return span == null ? NOOP : span; + } catch (RuntimeException err) { + failed(err); + return NOOP; + } + } + + /** + * Turns span creation off, or back on, for the calling thread. The exporter + * uses it so its own requests to the collector are not traced -- each export + * would otherwise produce a span, and exporting that one another. + */ + public static void setSuppressed(boolean suppressed) { + SUPPRESSED.set(suppressed ? Boolean.TRUE : null); + } + + /** + * Applies the rules {@link Web} enforces on request header lines to lines an + * exporter will send, so a tracer can refuse a bad configuration when it is + * opened. Otherwise the server starts, and every export then fails on the + * same check. One rule set, not a copy of it. + * + * @param lines {@code "Name: value"} strings + * @throws java.io.IOException naming the first line that could not be sent; + * the message never quotes a value, since values carry credentials + */ + public static void checkHeaderLines(List lines) throws java.io.IOException { + HeaderLines.validate(lines); + } + + /** Whether {@link #setSuppressed} is in force on this thread. */ + public static boolean isSuppressed() { + return SUPPRESSED.get() != null; + } + + /** + * Names the current server span after the route that matched. Called by the + * generated routers, which are the only code that knows the TEMPLATE -- the + * path alone would make every pet id its own operation. + */ + public static void route(String template) { + // Every generated router calls this on every matched request, traced or + // not; with no tracer that is this one read and nothing else. + if(tracer == null) { + return; + } + Span span = currentOrNull(); + if(span == null || template == null) { + return; + } + try { + // Inside the guard: the generated routers call this on every matched + // request, so a tracer whose getKind throws would otherwise turn a + // request that succeeded into a 500. + if(span.getKind() != Span.KIND_SERVER) { + return; + } + if(span.isRecording()) { + span.setAttribute("http.route", template); + } + String name = span.getName(); + // The name is the method until a route is known. Only the first match + // renames it, so two chained routers cannot append twice. + if(name != null && name.indexOf(' ') < 0) { + span.updateName(name + " " + template); + } + } catch (RuntimeException err) { + failed(err); + } + } + + // ------------------------------------------------------------------ + // Hooks the runtime calls. Each one is a single null test when tracing is + // off. + // ------------------------------------------------------------------ + + /** + * The span for one request, made current. Everything is read from the + * request NOW, because a Request is valid only while its handler runs. + */ + static Span startServer(HttpServer.Request request, boolean secure) { + Tracer t = tracer; + if(t == null || request == null) { + return null; + } + Span span = null; + try { + String method = request.getMethod(); + span = t.startSpan(method == null ? "HTTP" : method, Span.KIND_SERVER, null, + request.getHeader(TRACEPARENT), request.getHeader(TRACESTATE)); + if(span == null) { + return null; + } + if(span.isRecording()) { + span.setAttribute("http.request.method", method); + String target = request.getTarget(); + if(target != null) { + // The PATH only. The query string is where tokens and + // personal data travel, and a trace backend is not where + // either belongs. + int query = target.indexOf('?'); + span.setAttribute("url.path", query < 0 ? target : target.substring(0, query)); + } + span.setAttribute("url.scheme", secure ? "https" : "http"); + String version = request.getVersion(); + if(version != null && version.startsWith("HTTP/")) { + span.setAttribute("network.protocol.version", version.substring(5)); + } + String host = request.getHeader("host"); + if(host != null) { + span.setAttribute("server.address", hostOnly(host)); + } + String agent = request.getHeader("user-agent"); + if(agent != null) { + span.setAttribute("user_agent.original", agent); + } + } + enter(span); + return span; + } catch (RuntimeException err) { + failed(err); + abandon(span); + return null; + } + } + + /** + * Ends a request's span once the response is written -- or failed to be -- and + * clears the current span for the next request on the thread. + * + * @param status the status sent, or -1 when the write failed + */ + static void endServer(Span span, int status, Throwable error) { + if(span == null) { + return; + } + try { + if(span.isRecording()) { + if(status > 0) { + span.setAttribute("http.response.status_code", (long)status); + } + if(error != null) { + span.recordException(error); + } else if(status >= 500) { + // Server spans fail on 5xx only. A 404 is this server answering + // correctly about something that is not there. + span.setError(String.valueOf(status)); + } else if(status < 0) { + span.setError("the response could not be written"); + } + } + } catch (RuntimeException err) { + failed(err); + } + finish(span); + // The request that stopped the server has now ended, and its span with it: + // only now can the tracer go without taking that span down unexported. + Tracer after = span.shutdownOnEnd; + if(after != null) { + span.shutdownOnEnd = null; + shutdown(after, span.shutdownOnEndMillis); + } + } + + /** + * Stops {@code owned} -- now, or, when this thread is serving a request, once + * that request's span has ended. A handler that calls Backend.stop() is exempt + * from the drain precisely so its response can still be written, and its span + * ends after that write; shutting the tracer down first made the exporter + * refuse that span, so every shutdown requested over HTTP lost its own trace. + */ + static void shutdownAfterServing(Tracer owned, int timeoutMillis) { + Span serving = servingSpan(); + if(serving != null) { + serving.shutdownOnEnd = owned; + serving.shutdownOnEndMillis = timeoutMillis; + return; + } + shutdown(owned, timeoutMillis); + } + + /** The server span of the request this thread is serving, or null. */ + private static Span servingSpan() { + Span span = currentOrNull(); + int depth = 0; + while(span != null && depth++ < 64) { + try { + if(span.getKind() == Span.KIND_SERVER) { + return span; + } + } catch (RuntimeException err) { + failed(err); + return null; + } + span = span.previous; + } + return null; + } + + /** + * An outbound HTTP span, made current while the call runs. Null when tracing + * is off, suppressed on this thread, or already inside a client span -- one + * outbound operation is one span, whatever it happens to be built from. + */ + static Span startHttpClient(String method, String url, List callerHeaders) { + Tracer t = tracer; + if(t == null || isSuppressed()) { + return null; + } + if(callerTraceparent(callerHeaders)) { + // The caller chose which trace this request belongs to, and the service + // it reaches joins that one. A span recorded here would sit in the + // CURRENT trace and describe a request whose context it never sent. + return null; + } + String verb = method == null ? "GET" : method; + Span span = begin(verb, Span.KIND_CLIENT, null, null); + if(span == null) { + return null; + } + try { + if(span.isRecording()) { + span.setAttribute("http.request.method", verb); + if(url != null) { + // Redacted the way every log line here redacts it: a presigned + // URL's query IS the credential, and userinfo is a password. + span.setAttribute("url.full", Urls.forMessage(url)); + String host = authority(url); + if(host != null) { + span.setAttribute("server.address", hostOnly(host)); + } + } + } + } catch (RuntimeException err) { + failed(err); + } + return span; + } + + /** + * The header lines that carry the trace to the service being called, or null + * for none. None when the caller already set a traceparent of its own: that + * is a deliberate choice about which trace the request belongs to. + */ + static List propagationHeaders(Span span, List callerHeaders) { + if(span == null) { + return null; + } + if(callerTraceparent(callerHeaders)) { + return null; + } + try { + String parent = span.traceparent(); + if(parent == null) { + return null; + } + List out = new ArrayList(2); + out.add(TRACEPARENT + ": " + parent); + String state = span.tracestate(); + if(state != null && state.length() > 0) { + out.add(TRACESTATE + ": " + state); + } + return out; + } catch (RuntimeException err) { + failed(err); + return null; + } + } + + /** + * Ends an outbound span. + * + * @param status the response status, or -1 when there was none + */ + static void endHttpClient(Span span, int status, Throwable error) { + if(span == null) { + return; + } + try { + if(span.isRecording()) { + if(status > 0) { + span.setAttribute("http.response.status_code", (long)status); + } + if(error != null) { + span.recordException(error); + } else if(status >= 400) { + // A client span fails on 4xx as well: the call did not get + // what it asked for, whoever was wrong. + span.setError(String.valueOf(status)); + } + } + } catch (RuntimeException err) { + failed(err); + } + finish(span); + } + + /** + * A span for one database statement, made current while it runs. + * + *

The statement is recorded as the application wrote it, placeholders and + * all. The bound VALUES never are: they are the rows. A deployment that also + * inlines literals into its SQL can drop the text with + * {@code cn1.otel.attributes.exclude=db.query.text}. + */ + static Span startDatabase(String system, String sql) { + Tracer t = tracer; + if(t == null || isSuppressed()) { + return null; + } + String operation = firstKeyword(sql); + Span span = begin(operation == null ? system : operation, Span.KIND_CLIENT, null, null); + if(span == null) { + return null; + } + try { + if(span.isRecording()) { + span.setAttribute("db.system", system); + span.setAttribute("db.system.name", system); + if(operation != null) { + span.setAttribute("db.operation.name", operation); + } + if(sql != null) { + span.setAttribute("db.query.text", sql); + } + } + } catch (RuntimeException err) { + failed(err); + } + return span; + } + + /** An integer attribute on a span, guarded like every other hook. */ + static void setAttribute(Span span, String key, long value) { + if(span == null) { + return; + } + try { + if(span.isRecording()) { + span.setAttribute(key, value); + } + } catch (RuntimeException err) { + failed(err); + } + } + + /** + * Whether the caller's own header lines already carry trace context -- a + * traceparent, or a tracestate alone. The state is part of the caller's + * context too: a traceparent of ours added beside it paired the caller's + * vendor state with an unrelated trace id, or, with state of our own, sent + * two tracestate headers. + */ + static boolean callerTraceparent(List callerHeaders) { + if(callerHeaders == null) { + return false; + } + for(int iter = 0 ; iter < callerHeaders.size() ; iter++) { + String line = String.valueOf(callerHeaders.get(iter)).trim(); + if(isHeaderLine(line, TRACEPARENT) || isHeaderLine(line, TRACESTATE)) { + return true; + } + } + return false; + } + + private static boolean isHeaderLine(String line, String name) { + return line.regionMatches(true, 0, name, 0, name.length()) + && line.length() > name.length() + && (line.charAt(name.length()) == ':' || line.charAt(name.length()) == ' '); + } + + /** Ends a statement's span. */ + static void endDatabase(Span span, Throwable error) { + if(span == null) { + return; + } + if(error != null) { + guardedException(span, error); + } + finish(span); + } + + /** + * The span for one Lambda invocation, made current. + * + *

The host hands the invocation's trace over in X-Ray's own format rather + * than W3C's; when that is all there is, it is translated so the invocation + * still joins the caller's trace. The two describe the same 128-bit id: X-Ray + * writes it as a version, 8 hex digits of time and 24 of randomness, and W3C + * as the 32 of them run together. + */ + static Span startLambda(String traceHeader, String requestId) { + Tracer t = tracer; + if(t == null) { + return null; + } + Span span = null; + try { + String name = System.getenv("AWS_LAMBDA_FUNCTION_NAME"); + span = t.startSpan(name == null ? "invoke" : name, Span.KIND_SERVER, null, + fromXRay(traceHeader), null); + if(span == null) { + return null; + } + if(span.isRecording()) { + span.setAttribute("cloud.provider", "aws"); + span.setAttribute("faas.trigger", "other"); + if(requestId != null) { + span.setAttribute("faas.invocation_id", requestId); + } + } + enter(span); + return span; + } catch (RuntimeException err) { + failed(err); + abandon(span); + return null; + } + } + + /** + * Ends an invocation's span. The runtime loop then flushes before it polls + * again, rather than here, so the result is posted without waiting on the + * collector. + */ + static void endLambda(Span span, Throwable error) { + endLambda(span, error, null); + } + + /** + * Ends an invocation span with what went wrong, in order: the invocation's own + * failure, then a failure to tell the host about it. Either may be null. + */ + static void endLambda(Span span, Throwable error, Throwable reporting) { + if(span == null) { + return; + } + if(error != null) { + guardedException(span, error); + } + if(reporting != null) { + guardedException(span, reporting); + } + finish(span); + } + + /** Flushes the installed tracer, if any. */ + static void flush(int timeoutMillis) { + Tracer t = tracer; + if(t == null) { + return; + } + try { + t.flush(timeoutMillis); + } catch (RuntimeException err) { + failed(err); + } + } + + /** + * Flushes, stops and uninstalls {@code owned}, the tracer of a server that is + * stopping. Three places it can be: + *

    + *
  • installed: taken out of the slot and stopped;
    • + *
  • displaced by a start-up still in progress, which holds it as the + * tracer to restore if it fails: taken out of that claim -- so a failed + * start-up cannot put back the tracer of a server that has stopped -- and + * stopped, since nothing is running it any more;
    • + *
  • neither: replaced by an install or a commit, which already stopped it, + * and whatever replaced it belongs to someone else.
    • + *
+ */ + static void shutdown(Tracer owned, int timeoutMillis) { + if(owned == null) { + return; + } + boolean stop = false; + synchronized(LIFECYCLE) { + if(tracer == owned) { + tracer = null; + stop = true; + } + for(int iter = 0 ; iter < PENDING.size() ; iter++) { + Swap open = (Swap)PENDING.get(iter); + if(open.previous == owned) { + open.previous = null; + stop = true; + } + } + } + if(!stop) { + return; + } + try { + owned.shutdown(timeoutMillis); + } catch (RuntimeException err) { + failed(err); + } + } + + /** The tracer's counters, when one is installed. */ + static void metrics(Map out) { + Tracer t = tracer; + if(t == null) { + return; + } + try { + t.metrics(out); + } catch (RuntimeException err) { + failed(err); + } + } + + // ------------------------------------------------------------------ + + private static Span currentOrNull() { + Object value = CURRENT.get(); + return value instanceof Span ? (Span)value : null; + } + + /** + * A child of the current span, made current. Null when tracing is off, + * suppressed, or when a CLIENT span is already current: an insert that runs a + * query of its own to learn its key is one statement to the caller, and an + * outbound call cannot have another outbound call inside it. + */ + private static Span begin(String name, int kind, String traceparent, String tracestate) { + Tracer t = tracer; + if(t == null || isSuppressed()) { + return null; + } + Span parent = currentOrNull(); + try { + // Inside the guard, like every other call into the tracer: this runs for + // every outbound call and statement, and a span whose getKind throws must + // not fail the operation it was only meant to observe. + if(kind == Span.KIND_CLIENT && parent != null && parent.getKind() == Span.KIND_CLIENT) { + return null; + } + Span span = t.startSpan(name, kind, parent, traceparent, tracestate); + if(span == null) { + return null; + } + enter(span); + return span; + } catch (RuntimeException err) { + failed(err); + return null; + } + } + + private static void enter(Span span) { + span.previous = currentOrNull(); + span.entered = true; + CURRENT.set(span); + } + + /** + * Makes {@code span} stop being this thread's current span WITHOUT ending it. + * HTTP/2 serves several streams in one turn and writes their responses after + * the last handler, so a span has to leave -- or the next stream's span would + * start as its child -- before it can end. + */ + static void leave(Span span) { + if(span != null && span.entered) { + span.entered = false; + CURRENT.set(span.previous); + span.previous = null; + } + } + + /** Ends a span and restores what was current before it. */ + private static void finish(Span span) { + if(span.entered) { + span.entered = false; + CURRENT.set(span.previous); + span.previous = null; + } + try { + span.end(); + } catch (RuntimeException err) { + failed(err); + } + } + + /** + * Finishes a span the tracer handed out before a later call into it failed. + * Dropping the reference instead would leave the tracer holding whatever state + * it keeps for an open span -- once per request, for as long as the fault lasts. + * Discarded, because what it recorded is incomplete; each step guarded, because + * the tracer is already known to be failing. + */ + private static void abandon(Span span) { + if(span == null) { + return; + } + try { + span.discard(); + } catch (RuntimeException err) { + // The tracer is already failing; ending the span below still matters. + failed(err); + } + try { + span.end(); + } catch (RuntimeException err) { + failed(err); + } + } + + private static void guardedException(Span span, Throwable error) { + try { + span.recordException(error); + } catch (RuntimeException err) { + failed(err); + } + } + + /** Once per process: a broken tracer must not also flood the log. */ + private static void failed(RuntimeException err) { + if(!reportedFailure) { + reportedFailure = true; + System.err.println("tracing failed and the operation continued untraced: " + err); + } + } + + /** "example.com:8080" to "example.com", "[::1]:80" to "::1". */ + static String hostOnly(String host) { + String h = host.trim(); + if(h.startsWith("[")) { + int close = h.indexOf(']'); + return close > 0 ? h.substring(1, close) : h; + } + int colon = h.indexOf(':'); + return colon >= 0 ? h.substring(0, colon) : h; + } + + /** The host[:port] of an absolute URL, without userinfo; null when there is none. */ + static String authority(String url) { + int scheme = url.indexOf("://"); + if(scheme < 0) { + return null; + } + int start = scheme + 3; + int end = url.length(); + for(int iter = start ; iter < url.length() ; iter++) { + char c = url.charAt(iter); + if(c == '/' || c == '?' || c == '#') { + end = iter; + break; + } + } + String authority = url.substring(start, end); + int at = authority.lastIndexOf('@'); + return at >= 0 ? authority.substring(at + 1) : authority; + } + + /** + * The SQL keyword a statement opens with, upper-cased, or null. Folded by + * hand: String.toUpperCase is locale sensitive, and on a Turkish host + * "insert" would come back with a dotted capital I. + */ + static String firstKeyword(String sql) { + if(sql == null) { + return null; + } + int at = 0; + int n = sql.length(); + while(at < n && (sql.charAt(at) <= ' ' || sql.charAt(at) == '(')) { + at++; + } + StringBuilder out = new StringBuilder(); + while(at < n && out.length() < 16) { + char c = sql.charAt(at); + if(c >= 'a' && c <= 'z') { + out.append((char)(c - 32)); + } else if(c >= 'A' && c <= 'Z') { + out.append(c); + } else { + break; + } + at++; + } + return out.length() == 0 ? null : out.toString(); + } + + /** + * X-Ray's {@code Root=1-5759e988-bd862e3fe1be46a994272793;Parent=53995c3f42cd8ad8;Sampled=1} + * as a W3C traceparent, or null when it is absent or not in that shape. + */ + static String fromXRay(String header) { + if(header == null) { + return null; + } + String root = null; + String parent = null; + String sampled = "0"; + int at = 0; + while(at < header.length()) { + int end = header.indexOf(';', at); + if(end < 0) { + end = header.length(); + } + String part = header.substring(at, end).trim(); + at = end + 1; + if(part.startsWith("Root=")) { + root = part.substring(5); + } else if(part.startsWith("Parent=")) { + parent = part.substring(7); + } else if(part.startsWith("Sampled=")) { + sampled = part.substring(8); + } + } + // 1-xxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx: version, 8 hex, 24 hex. + if(root == null || parent == null || root.length() != 35 || !root.startsWith("1-") + || root.charAt(10) != '-' || parent.length() != 16) { + return null; + } + String traceId = root.substring(2, 10) + root.substring(11); + if(!isLowerHex(traceId) || !isLowerHex(parent)) { + return null; + } + return "00-" + traceId + "-" + parent + "-" + ("1".equals(sampled) ? "01" : "00"); + } + + private static boolean isLowerHex(String value) { + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + if(!(c >= '0' && c <= '9') && !(c >= 'a' && c <= 'f')) { + return false; + } + } + return true; + } + + /** What every hook answers with when there is no tracer. */ + private static final class NoopSpan extends Span { + public Span setAttribute(String key, String value) { + return this; + } + + public Span setAttribute(String key, long value) { + return this; + } + + public Span setAttribute(String key, double value) { + return this; + } + + public Span setAttribute(String key, boolean value) { + return this; + } + + public Span recordException(Throwable error) { + return this; + } + + public Span setError(String description) { + return this; + } + + public Span updateName(String name) { + return this; + } + + public String getName() { + return ""; + } + + public int getKind() { + return KIND_INTERNAL; + } + + public boolean isRecording() { + return false; + } + + public String traceparent() { + return null; + } + + public String tracestate() { + return null; + } + + public void discard() { + } + + public void end() { + } + } +} diff --git a/vm/backend/src/com/codename1/backend/annotations/OpenTelemetry.java b/vm/backend/src/com/codename1/backend/annotations/OpenTelemetry.java new file mode 100644 index 00000000000..d5cad57d334 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/OpenTelemetry.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Traces this server with OpenTelemetry. +/// +/// Put it on any class in the backend module -- a controller is the natural +/// place. The build then wires a tracer into the entry point it generates, and +/// from then on every request, every outbound `Web` call and every `Database` +/// statement is a span, exported over OTLP/HTTP to the collector the deployment +/// names: +/// +/// ```java +/// @OpenTelemetry(serviceName = "orders") +/// @RestController +/// public class OrderController { ... } +/// ``` +/// +/// ``` +/// OTEL_EXPORTER_OTLP_ENDPOINT=https://collector.internal:4318 ./server +/// ``` +/// +/// Nothing else changes. An incoming `traceparent` makes the request part of the +/// caller's trace -- which is how a Codename One app's own spans connect to the +/// server's -- and the W3C trace context is sent on every outbound request. +/// +/// A project that would rather not touch its source can set +/// `cn1.otel.enabled=true` in `application.properties` instead; the effect is +/// the same. +/// +/// THIS IS A BUILD-TIME SWITCH, and deliberately so. Without it the tracer is +/// never referenced, so the translator leaves it out of the binary entirely. With +/// it, the deployment still has the last word: `OTEL_SDK_DISABLED=true` turns +/// tracing off at start-up without a rebuild. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.TYPE) +public @interface OpenTelemetry { + /// The `service.name` spans are reported under, unless `OTEL_SERVICE_NAME` + /// or `cn1.otel.service.name` names another. Empty means `unknown_service`, + /// which is what every OpenTelemetry SDK reports when nobody said. + String serviceName() default ""; +} diff --git a/vm/backend/src/com/codename1/backend/otel/BatchExporter.java b/vm/backend/src/com/codename1/backend/otel/BatchExporter.java new file mode 100644 index 00000000000..cadd91163d2 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/otel/BatchExporter.java @@ -0,0 +1,615 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.otel; + +import com.codename1.backend.Tracing; +import com.codename1.backend.Web; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Batches ended spans and posts them to the collector, on a thread of its own. + * + *

A PLATFORM THREAD, never a virtual one, and never the request's. Outbound + * HTTP on the packaged runtime is libcurl, which blocks the OS thread it runs on + * -- and a virtual thread's OS thread is the host carrying every other connection + * pinned to it. An export done inline would stall all of them for as long as the + * collector took to answer; one done on a virtual thread would do the same to + * whichever host it landed on. Here a slow or absent collector costs this thread + * and nothing else. + * + *

BOUNDED. The queue holds at most {@code maxQueue} spans, and a span that + * arrives when it is full is dropped and counted. A collector that is down must + * degrade to missing traces, never to a server that runs out of memory holding + * them. Relayed client payloads are bounded the same way, by bytes. + * + *

The request path only ever takes the lock for an append. + */ +final class BatchExporter implements Runnable { + /** Relayed payloads posted per round before the server's own spans get a turn. */ + static final int RELAY_PER_ROUND = 8; + + private final Object lock = new Object(); + private final String endpoint; + private final List headers; + private final boolean protobuf; + private final Map resource; + private final int maxQueue; + private final int maxBatch; + private final long delayMillis; + private final long maxRelayBytes; + + private ArrayList queue = new ArrayList(); + /** Each an Object[] {byte[] body, String contentType}: payloads the relay accepted. */ + private ArrayList relayed = new ArrayList(); + private long relayedBytes; + /* + * Watermarks for flush(). Items leave each queue in order, so "everything + * queued before the flush has gone" is exactly "the drained count has reached + * the enqueued count the flush saw" -- which stays true on a busy server whose + * queue is never empty, where waiting for an empty queue would not. + */ + private long enqueuedSpans; + private long drainedSpans; + private long enqueuedRelayed; + private long drainedRelayed; + /** The furthest watermarks any waiting flush asked for; the thread exports promptly until it reaches them. */ + private long flushSpans; + private long flushRelayed; + private boolean stopping; + private boolean stopped; + /** + * Nothing is posted before this time: set after a retryable failure (429, a + * 5xx gateway answer, no connection) so the retry waits out a backoff. Waited + * for on the lock, never slept through -- shutdown's notifyAll ends it at once. + */ + private long retryAt; + private Thread thread; + + // Counters for the metrics snapshot. Written under the lock. + private long exportedSpans; + private long droppedSpans; + private long failedExports; + private long rejectedSpans; + /** Spans the collector rejected in the last successful POST; see post(). */ + private long lastRejected; + private long relayedPayloads; + private long droppedRelayed; + private String lastError; + + BatchExporter(String endpoint, List headers, boolean protobuf, Map resource, + int maxQueue, int maxBatch, long delayMillis, long maxRelayBytes) { + this.endpoint = endpoint; + this.headers = headers; + this.protobuf = protobuf; + this.resource = resource; + this.maxQueue = maxQueue; + this.maxBatch = maxBatch; + this.delayMillis = delayMillis; + this.maxRelayBytes = maxRelayBytes; + } + + boolean isProtobuf() { + return protobuf; + } + + void start() { + Thread t = new Thread(this, "cn1-otel-exporter"); + // So a JVM run of the server can exit: the exporter must never be the + // reason a process that finished stays up. + t.setDaemon(true); + thread = t; + t.start(); + } + + /** From the request path, as a span ends. */ + void add(OtelSpan span) { + synchronized(lock) { + // STOPPING counts as closed, not just stopped: shutdown() has already + // dropped the queue, and a span that ends after it -- a request still + // in flight when its tracer was replaced -- would otherwise be taken + // and exported by a worker that is meant to be finishing. + if(stopped || stopping || queue.size() >= maxQueue) { + droppedSpans++; + return; + } + queue.add(span); + enqueuedSpans++; + if(queue.size() >= maxBatch) { + lock.notifyAll(); + } + } + } + + /** + * From the relay: a client's export, already re-encoded. False when the + * relay's byte budget is spent, which the relay answers with 503 so the client + * backs off rather than resending into a full queue. + */ + boolean addRelayed(byte[] body, String contentType) { + synchronized(lock) { + if(stopped || stopping || relayedBytes + body.length > maxRelayBytes) { + droppedRelayed++; + return false; + } + // The third slot marks a payload that has had its one retry. + relayed.add(new Object[] {body, contentType, null}); + relayedBytes += body.length; + enqueuedRelayed++; + lock.notifyAll(); + return true; + } + } + + /** Blocks until everything queued before the call is exported, or the time is up. */ + void flush(int timeoutMillis) { + long deadline = System.currentTimeMillis() + Math.max(0, timeoutMillis); + synchronized(lock) { + if(thread == null || stopped) { + return; + } + long spans = enqueuedSpans; + long payloads = enqueuedRelayed; + flushSpans = Math.max(flushSpans, spans); + flushRelayed = Math.max(flushRelayed, payloads); + lock.notifyAll(); + while((drainedSpans < spans || drainedRelayed < payloads) && !stopped) { + long left = deadline - System.currentTimeMillis(); + if(left <= 0) { + return; + } + try { + lock.wait(left); + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + return; + } + } + } + } + + /** + * Flushes for at most {@code timeoutMillis}, then stops the thread. What the + * flush could not export in that time is DROPPED and counted, not drained: + * waiting for an empty queue kept a replaced tracer posting every remaining + * batch -- retries included -- with its old credentials for as long as a slow + * or absent collector took, while its caller had been promised a bounded stop. + * + *

What is bounded is THIS exporter's work: after the window nothing new is + * posted, nothing is retried, and every later span or payload is refused. A + * POST already in flight is not cancelled; it ends on Web's own timeouts -- + * connect, and a low-speed or per-read limit -- which a collector trickling + * bytes can stretch. That is a property of Web, which offers no cancellation + * and no total deadline on either transport, and it belongs there rather than + * worked around from one caller: the worker is a daemon thread, the request + * it is in was the last it will make, and it exits when that returns. + */ + void shutdown(int timeoutMillis) { + flush(timeoutMillis); + synchronized(lock) { + stopping = true; + droppedSpans += queue.size(); + queue.clear(); + droppedRelayed += relayed.size(); + relayed.clear(); + relayedBytes = 0; + lock.notifyAll(); + } + } + + private boolean isStopping() { + synchronized(lock) { + return stopping; + } + } + + void metrics(Map out) { + synchronized(lock) { + out.put("tracing", stopped ? "stopped" : "on"); + out.put("spansExported", Long.valueOf(exportedSpans)); + out.put("spansDropped", Long.valueOf(droppedSpans)); + out.put("spansQueued", Integer.valueOf(queue.size())); + out.put("traceExportsFailed", Long.valueOf(failedExports)); + if(rejectedSpans > 0) { + out.put("spansRejected", Long.valueOf(rejectedSpans)); + } + if(relayedPayloads > 0 || droppedRelayed > 0) { + out.put("clientExportsRelayed", Long.valueOf(relayedPayloads)); + out.put("clientExportsDropped", Long.valueOf(droppedRelayed)); + } + if(lastError != null) { + out.put("traceExportLastError", lastError); + } + } + } + + public void run() { + // The exporter's own requests are not traced: each would be a span, and + // exporting that span another. + Tracing.setSuppressed(true); + long nextTick = System.currentTimeMillis() + delayMillis; + while(true) { + List batch; + List payloads; + synchronized(lock) { + while(!stopping) { + long now = System.currentTimeMillis(); + long until; + if(now < retryAt) { + // Backing off: nothing goes before retryAt, a pending + // flush included, or a flush would hammer the collector + // that just asked for less. + until = retryAt; + } else if(queue.size() >= maxBatch || !relayed.isEmpty() + || drainedSpans < flushSpans || drainedRelayed < flushRelayed + || now >= nextTick) { + break; + } else { + until = nextTick; + } + try { + lock.wait(Math.max(1, until - now)); + } catch (InterruptedException err) { + // Only a shutdown interrupts this thread; carry on to it. + } + } + batch = take(queue, maxBatch); + // A bounded share of the relay per round, between local batches. + // Draining it all at once let clients posting many tiny payloads + // to a slow collector hold this thread for as long as that took, + // while the server's own spans overflowed their queue and were + // dropped. + payloads = take(relayed, RELAY_PER_ROUND); + for(int iter = 0 ; iter < payloads.size() ; iter++) { + relayedBytes -= ((byte[])((Object[])payloads.get(iter))[0]).length; + } + } + if(System.currentTimeMillis() >= nextTick) { + nextTick = System.currentTimeMillis() + delayMillis; + } + int requeuedSpans = 0; + if(!batch.isEmpty()) { + if(isStopping()) { + // Taken off the queue before shutdown's window closed, posted + // after it: a request with the retired tracer's endpoint and + // credentials after shutdown had returned. Dropped instead, the + // way the relay loop drops the rest of its round. + synchronized(lock) { + droppedSpans += batch.size(); + } + } else { + requeuedSpans = exportSpans(batch); + } + } + int requeuedPayloads = 0; + for(int iter = 0 ; iter < payloads.size() ; iter++) { + if(isStopping()) { + // The rest of this round goes the way shutdown() sends the + // queue: dropped, so the stop stays bounded. + synchronized(lock) { + droppedRelayed += payloads.size() - iter; + } + break; + } + Object[] payload = (Object[])payloads.get(iter); + int result = post((byte[])payload[0], (String)payload[1]); + synchronized(lock) { + if(result == SENT) { + relayedPayloads++; + } else if(result == RETRY) { + // The collector asked for less: the backoff starts whatever + // becomes of this payload, and the rest of the round waits + // it out rather than being posted into the same trouble. + retryAt = System.currentTimeMillis() + delayMillis; + boolean retryThis = payload[2] == null; + if(!retryThis) { + droppedRelayed++; // its one retry is spent + } + int rest = retryThis ? iter : iter + 1; + if(requeueRelayed(payloads, rest, retryThis)) { + requeuedPayloads = payloads.size() - rest; + } else { + droppedRelayed += payloads.size() - rest; + } + break; + } else { + droppedRelayed++; + } + } + } + synchronized(lock) { + // A requeued span is not drained: flush() is waiting for it to be + // exported or dropped, which happens on its retry. + drainedSpans += batch.size() - requeuedSpans; + drainedRelayed += payloads.size() - requeuedPayloads; + lock.notifyAll(); + if(stopping && queue.isEmpty() && relayed.isEmpty()) { + stopped = true; + lock.notifyAll(); + return; + } + } + } + } + + private static List take(ArrayList from, int max) { + int n = Math.min(from.size(), max); + List out = new ArrayList(n); + for(int iter = 0 ; iter < n ; iter++) { + out.add(from.get(iter)); + } + // Removed from the FRONT in one step; a remove(0) per span would shift the + // rest of the queue every time. + from.subList(0, n).clear(); + return out; + } + + /** + * Posts one batch. A retryable failure puts the spans that have not been + * retried yet back at the head of the queue for one more try after the + * backoff; the rest are dropped. + * + * @return how many spans went back on the queue + */ + private int exportSpans(List batch) { + byte[] body; + try { + Map request = OtlpTracer.exportRequest(resource, batch); + body = protobuf ? OtlpSchema.protobuf(request) : OtlpSchema.json(request); + } catch (Exception err) { + // A span this code built and cannot encode is a bug here, not in the + // collector; count it and keep exporting the rest. + synchronized(lock) { + failedExports++; + droppedSpans += batch.size(); + lastError = bounded("encode: " + err.getMessage()); + } + return 0; + } + int result = post(body, protobuf ? "application/x-protobuf" : "application/json"); + synchronized(lock) { + if(result == SENT) { + // A 200 can still carry a partial success: the collector names how + // many spans it dropped, and those were not exported. + long rejected = Math.min(lastRejected, batch.size()); + exportedSpans += batch.size() - rejected; + rejectedSpans += rejected; + return 0; + } + if(result == RETRY) { + // On EVERY retryable answer, whether or not these spans can go + // back: dropping a batch without it let the next one go straight + // out into the same 429, unthrottled. + retryAt = System.currentTimeMillis() + delayMillis; + } + if(result != RETRY || stopping) { + droppedSpans += batch.size(); + return 0; + } + // Once per span, never more: an overloaded collector is not helped by + // this server hammering it, and what is queued behind these keeps + // arriving. Room permitting -- a full queue is the bound that holds. + List again = new ArrayList(); + for(int iter = 0 ; iter < batch.size() ; iter++) { + OtelSpan span = (OtelSpan)batch.get(iter); + if(!span.exportRetried) { + span.exportRetried = true; + again.add(span); + } + } + if(again.isEmpty() || queue.size() + again.size() > maxQueue) { + droppedSpans += batch.size(); + return 0; + } + droppedSpans += batch.size() - again.size(); + queue.addAll(0, again); + return again.size(); + } + } + + /** + * Puts payloads[from..] back at the head of the relay queue -- payloads[from] + * marked as retried when {@code markFirst}, since its POST was the one that + * failed -- or answers false, having changed nothing, when the relay's byte + * budget has no room for them. The caller starts the backoff. Called holding + * the lock. + */ + private boolean requeueRelayed(List payloads, int from, boolean markFirst) { + if(stopping) { + return false; + } + long bytes = 0; + for(int iter = from ; iter < payloads.size() ; iter++) { + bytes += ((byte[])((Object[])payloads.get(iter))[0]).length; + } + if(relayedBytes + bytes > maxRelayBytes) { + return false; + } + List again = new ArrayList(); + for(int iter = from ; iter < payloads.size() ; iter++) { + Object[] payload = (Object[])payloads.get(iter); + // Only the one that was POSTED has used its retry. The rest of the + // round were never sent, and marking them too dropped each of them on + // its first real failure. + again.add(markFirst && iter == from + ? new Object[] {payload[0], payload[1], Boolean.TRUE} : payload); + } + relayed.addAll(0, again); + relayedBytes += bytes; + return true; + } + + private static final int SENT = 0; + /** A failure OTLP/HTTP defines as retryable: 429, 502-504, or no connection. */ + private static final int RETRY = 1; + private static final int FAILED = 2; + + /** + * One POST, and only one. There used to be a second, after a one-second + * Thread.sleep in here: a thread parked doing nothing, and a window in which + * a shutdown that had already given up waiting watched it post again with the + * old credentials. The retry now goes back on the queue instead -- see + * exportSpans -- and waits out its backoff on the lock, which shutdown wakes. + */ + private int post(byte[] body, String contentType) { + lastRejected = 0; + List lines = new ArrayList(headers.size() + 1); + lines.add("Content-Type: " + contentType); + lines.addAll(headers); + int status; + try { + Web.Result result = Web.request("POST", endpoint, lines, body); + status = result.getStatus(); + if(status >= 200 && status < 300) { + lastRejected = partialSuccess(result, contentType); + } + } catch (Exception err) { + recordFailure("could not reach the collector: " + err.getMessage()); + status = -1; + } + if(status >= 200 && status < 300) { + return SENT; + } + if(status > 0) { + recordFailure("the collector answered " + status); + } + boolean retryable = status < 0 || status == 429 || status == 502 || status == 503 + || status == 504; + return retryable ? RETRY : FAILED; + } + + /** + * The OTLP partial-success answer: an ExportTraceServiceResponse whose + * partial_success reports rejected spans and why. A collector answers 200 for a + * batch it only partly accepted, so without reading this every dropped span was + * counted as exported and nobody could tell. Decoded in whichever encoding the + * collector answered in; anything unreadable is taken as full success, which is + * what a 200 without the field means. + * + * @return the number of spans rejected, 0 for none + */ + private long partialSuccess(Web.Result result, String requestType) { + byte[] body = result.getBody(); + if(body == null || body.length == 0) { + return 0; + } + String type = result.getHeader("content-type"); + boolean json = type != null ? type.regionMatches(true, 0, "application/json", 0, 16) + : !requestType.startsWith("application/x-protobuf"); + long[] rejected = new long[1]; + String[] message = new String[1]; + try { + if(json) { + OtlpSchema.jsonPartialSuccess(result.getBodyAsString(), rejected, message); + } else { + OtlpSchema.protobufPartialSuccess(body, rejected, message); + } + } catch (Exception err) { + return 0; + } + if(rejected[0] > 0 || (message[0] != null && message[0].length() > 0)) { + recordFailure("the collector rejected " + rejected[0] + " span(s)" + + (message[0] == null || message[0].length() == 0 ? "" : ": " + message[0])); + } + return rejected[0] < 0 ? 0 : rejected[0]; + } + + /** + * Counted always, printed once per hundred. A missing collector is common -- + * a developer laptop, a misconfigured deployment -- and a line per batch would + * bury everything else the server logs. + */ + /** The longest diagnostic kept or logged. */ + static final int MAX_ERROR_CHARS = 512; + + /** + * {@code message} cut to {@link #MAX_ERROR_CHARS}. Much of it is the + * collector's own text -- a partial-success errorMessage, an exception naming + * what it read -- and a response may be megabytes, which lastError then held + * for the exporter's lifetime and the log printed as one line. + */ + static String bounded(String message) { + if(message == null || message.length() <= MAX_ERROR_CHARS) { + return message; + } + int end = MAX_ERROR_CHARS; + if(Character.isHighSurrogate(message.charAt(end - 1))) { + end--; + } + return message.substring(0, end) + "... (" + message.length() + " chars)"; + } + + private void recordFailure(String message) { + message = bounded(message); + long count; + synchronized(lock) { + failedExports++; + lastError = message; + count = failedExports; + } + if(count == 1 || count % 100 == 0) { + System.err.println("trace export to " + redact(endpoint) + " failed (" + count + + " so far): " + message); + } + } + + /** + * The endpoint as it may be logged: no query or fragment, where a token would travel, and + * no userinfo, where a password would. Both are how collectors are commonly + * authenticated, and this string goes into every failure line and every + * refused-configuration message. + */ + static String redact(String url) { + // The query AND the fragment, whichever comes first: an OAuth token rides + // in a fragment as readily as in a query, and cutting only at '?' logged it. + int cut = url.length(); + int query = url.indexOf('?'); + int fragment = url.indexOf('#'); + if(query >= 0) { + cut = query; + } + if(fragment >= 0 && fragment < cut) { + cut = fragment; + } + String out = url.substring(0, cut); + int scheme = out.indexOf("://"); + if(scheme >= 0) { + // Through the LAST '@' before the path: in "alice:secret@tenant@host" + // the first '@' is part of the password, and cutting there logged + // "tenant@". Only '/' ends the authority here, not also a backslash, + // though a browser reads one as a slash: java.net.URL and libcurl do + // not, so what a backslash separates may be userinfo, and a redactor + // that hides too much is the safe way to be wrong. + int start = scheme + 3; + int slash = out.indexOf('/', start); + int at = out.substring(start, slash < 0 ? out.length() : slash).lastIndexOf('@'); + if(at >= 0) { + out = out.substring(0, start) + "@" + out.substring(start + at + 1); + } + } + // Says something was there, and which part, without saying what. + return cut < url.length() ? out + url.charAt(cut) + "" : out; + } +} diff --git a/vm/backend/src/com/codename1/backend/otel/OtelSpan.java b/vm/backend/src/com/codename1/backend/otel/OtelSpan.java new file mode 100644 index 00000000000..c7077d850dd --- /dev/null +++ b/vm/backend/src/com/codename1/backend/otel/OtelSpan.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.otel; + +import com.codename1.backend.Span; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * A span as {@link OtlpTracer} records it. + * + *

Written by one thread -- the one doing the work it times -- and handed to + * the exporter only by {@link #end}, through a synchronized queue, so nothing + * here needs a lock of its own. + */ +final class OtelSpan extends Span { + /** Whether an export of this span has already been retried once; see BatchExporter. */ + boolean exportRetried; + /** OpenTelemetry's default attribute and event limits. */ + static final int MAX_ATTRIBUTES = 128; + static final int MAX_EVENTS = 128; + /** Past this a string attribute is truncated. A request body in an attribute is a bug. */ + static final int MAX_VALUE_LENGTH = 4096; + /** Past this an attribute KEY is dropped; see put. */ + static final int MAX_KEY_LENGTH = 256; + + private final OtlpTracer tracer; + final long traceHi; + final long traceLo; + final long spanId; + /** 0 for a root span. */ + final long parentId; + final boolean parentRemote; + final boolean sampled; + final String tracestate; + final int kind; + String name; + final long startEpochNanos; + /** + * The clock this span reads: an epoch time and the monotonic reading taken at + * the same moment, shared by every span of one trace in this process. + */ + final long anchorEpochNanos; + final long anchorNano; + long endEpochNanos; + /** Key to value (String, Long, Double or Boolean), in the order first set. */ + final Map attributes; + int droppedAttributes; + /** Each an Object[] {Long time, String name, Map attributes}. */ + final List events; + int droppedEvents; + /** 0 unset, 2 error: OTLP's StatusCode. OK is never set by instrumentation. */ + int statusCode; + String statusMessage; + private boolean ended; + private boolean discarded; + + OtelSpan(OtlpTracer tracer, String name, int kind, long traceHi, long traceLo, long spanId, + long parentId, boolean parentRemote, boolean sampled, String tracestate, + OtelSpan localParent) { + this.tracer = tracer; + // Bounded like every other string a span keeps: a caller-derived name + // could otherwise be any size, and the queue is bounded by span COUNT. + this.name = name == null ? "" : bound(name); + this.kind = kind; + this.traceHi = traceHi; + this.traceLo = traceLo; + this.spanId = spanId; + this.parentId = parentId; + this.parentRemote = parentRemote; + this.sampled = sampled; + this.tracestate = tracestate; + // ONE CLOCK PER TRACE. A root span anchors the wall clock to the monotonic + // one, and its descendants inherit that anchor and measure from it. Each + // span reading the wall clock for itself put spans a millisecond apart + // (the wall clock's resolution) on timelines that disagreed, and a child + // was reported ending after the parent that contains it. The monotonic + // reading also keeps a clock step from producing a span that ends before + // it starts. + if(localParent != null) { + this.anchorEpochNanos = localParent.anchorEpochNanos; + this.anchorNano = localParent.anchorNano; + } else { + this.anchorEpochNanos = System.currentTimeMillis() * 1000000L; + this.anchorNano = System.nanoTime(); + } + this.startEpochNanos = nowEpochNanos(); + this.attributes = sampled ? new LinkedHashMap() : null; + this.events = sampled ? new ArrayList(0) : null; + } + + public Span setAttribute(String key, String value) { + if(value != null && value.length() > MAX_VALUE_LENGTH) { + value = bound(value); + } + return put(key, value); + } + + public Span setAttribute(String key, long value) { + return put(key, Long.valueOf(value)); + } + + public Span setAttribute(String key, double value) { + if(Double.isNaN(value) || Double.isInfinite(value)) { + // JSON has no spelling for either, so the whole export would be + // refused by a collector over one attribute. Kept as text instead. + return put(key, String.valueOf(value)); + } + return put(key, Double.valueOf(value)); + } + + public Span setAttribute(String key, boolean value) { + return put(key, value ? Boolean.TRUE : Boolean.FALSE); + } + + private Span put(String key, Object value) { + if(!sampled || ended || key == null || value == null || tracer.excluded(key)) { + return this; + } + // An oversized KEY is dropped rather than cut. Values are truncated because a + // prefix of a value is still that value; a prefix of a key is a different + // attribute, and two long keys sharing one would silently overwrite each other. + // Left unbounded, one key read from a request could carry the whole export + // batch past the collector's size limit and lose every span in it. + if(key.length() > MAX_KEY_LENGTH) { + droppedAttributes++; + return this; + } + if(!attributes.containsKey(key) && attributes.size() >= MAX_ATTRIBUTES) { + droppedAttributes++; + return this; + } + attributes.put(key, value); + return this; + } + + /** Whether {@code owner} made this span. */ + boolean isFrom(OtlpTracer owner) { + return tracer == owner; + } + + public Span recordException(Throwable error) { + if(!sampled || ended || error == null) { + return this; + } + // The exclusion list governs these like any other attribute: it is how a + // deployment keeps failure details -- a message quoting user input, say -- + // out of its traces. That covers the status description too, which would + // otherwise carry the excluded message anyway. + String type = tracer.excluded("exception.type") ? null : error.getClass().getName(); + String message = tracer.excluded("exception.message") ? null : error.getMessage(); + statusCode = 2; + // Bounded like the event attribute below. The export queue is bounded by + // span COUNT, so an unbounded message per failing request is how that bound + // stops bounding memory. + String description = message == null ? type : message; + statusMessage = description == null ? null : bound(description); + if(events.size() >= MAX_EVENTS) { + droppedEvents++; + return this; + } + Map attrs = new LinkedHashMap(); + if(type != null) { + attrs.put("exception.type", type); + } + if(message != null) { + attrs.put("exception.message", message.length() > MAX_VALUE_LENGTH + ? bound(message) : message); + } + events.add(new Object[] {Long.valueOf(nowEpochNanos()), "exception", attrs}); + return this; + } + + public Span setError(String description) { + if(!sampled || ended) { + return this; + } + statusCode = 2; + statusMessage = description == null ? null : bound(description); + return this; + } + + public Span updateName(String newName) { + if(!ended && newName != null) { + name = bound(newName); + } + return this; + } + + public String getName() { + return name; + } + + public int getKind() { + return kind; + } + + public boolean isRecording() { + return sampled && !ended && !discarded; + } + + public String traceparent() { + return TraceContext.format(traceHi, traceLo, spanId, sampled); + } + + public String tracestate() { + return tracestate; + } + + public void discard() { + discarded = true; + } + + public void end() { + if(ended) { + return; + } + ended = true; + endEpochNanos = nowEpochNanos(); + if(sampled && !discarded) { + tracer.ended(this); + } + } + + /** + * At most MAX_VALUE_LENGTH chars, cut on a code point boundary. A cut through a + * surrogate pair kept a lone high surrogate, which UTF-8 encoding then + * replaced, so the exported value was not the one the server recorded. + */ + static String bound(String value) { + if(value.length() <= MAX_VALUE_LENGTH) { + return value; + } + int end = MAX_VALUE_LENGTH; + if(Character.isHighSurrogate(value.charAt(end - 1))) { + end--; + } + return value.substring(0, end); + } + + private long nowEpochNanos() { + long elapsed = System.nanoTime() - anchorNano; + return anchorEpochNanos + (elapsed < 0 ? 0 : elapsed); + } +} diff --git a/vm/backend/src/com/codename1/backend/otel/OtlpRelay.java b/vm/backend/src/com/codename1/backend/otel/OtlpRelay.java new file mode 100644 index 00000000000..792ec9c1cf6 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/otel/OtlpRelay.java @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.otel; + +import com.codename1.backend.Crypto; +import com.codename1.backend.HttpServer; +import com.codename1.backend.Json; +import com.codename1.backend.Tracing; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Accepts the app's spans and forwards them to the collector this server + * exports to. + * + *

This is what lets a mobile app report spans without shipping the + * collector's credentials in its binary -- anything inside an app package is + * public -- and without the collector having to answer a browser's CORS + * preflight. The app posts OTLP/JSON to its own backend, the way it already + * talks to it, and the backend adds the ingest token on the way out. + * + *

JSON in, because the server hands a handler its request body as text and a + * binary protobuf body would not survive that. What goes out is re-encoded, in + * whichever protocol this server exports with, from a tree rebuilt against the + * OTLP schema: a field the schema does not name is dropped, a malformed id is + * refused, and only then does the payload join the export queue. The relay is a + * public endpoint, so it treats what it receives as input, not as a message to + * pass along. + * + *

It answers as soon as the payload is queued. Forwarding inside the request + * would block the connection's host thread on the collector (see + * {@link BatchExporter}), and the client does not need to wait for the collector + * to hear about its own spans. + */ +final class OtlpRelay implements HttpServer.Handler { + private final byte[] path; + private final String pathText; + private final String token; + private final int maxBytes; + private final int maxSpans; + private final String corsOrigin; + private final BatchExporter exporter; + + /** The header a client puts the relay token in. */ + static final String TOKEN_HEADER = "x-cn1-telemetry-token"; + + OtlpRelay(String path, String token, int maxBytes, int maxSpans, String corsOrigin, + BatchExporter exporter) { + this.pathText = path; + this.path = ascii(path); + this.token = token == null || token.length() == 0 ? null : token; + this.maxBytes = maxBytes; + this.maxSpans = maxSpans; + this.corsOrigin = corsOrigin == null || corsOrigin.length() == 0 ? null : corsOrigin; + this.exporter = exporter; + } + + public HttpServer.Response handle(HttpServer.Request request) throws Exception { + if(!request.pathIs(path)) { + return null; + } + // The relay's own request is not a trace anyone asked for: a client + // exporting every few seconds would otherwise fill the backend's traces + // with the exports themselves. + Tracing.current().discard(); + String method = request.getMethod(); + if("OPTIONS".equals(method)) { + return preflight(); + } + if(!"POST".equals(method)) { + return answer(405, "POST OTLP/JSON to " + pathText); + } + if(token != null) { + String offered = request.getHeader(TOKEN_HEADER); + // UTF-8 on both sides, not the ASCII folding used for fixed replies: + // that mapped every non-ASCII character to '?', so distinct tokens + // compared equal: "s?cret" opened a relay whose token had an accented e. + if(offered == null || !Crypto.equalsConstantTime(utf8(offered), utf8(token))) { + return answer(401, "missing or wrong " + TOKEN_HEADER); + } + } + String type = request.getHeader("content-type"); + if(type == null || !type.regionMatches(true, 0, "application/json", 0, 16)) { + return answer(415, "the relay accepts application/json"); + } + String body = request.getBody(); + if(body == null || body.length() == 0) { + return answer(400, "empty export"); + } + // Measured in UTF-8 bytes, as the setting is. Counting characters let a + // body of three-byte characters through at three times the ceiling. + if(utf8Length(body, maxBytes) > maxBytes) { + return answer(413, "export larger than " + maxBytes + " bytes"); + } + byte[] encoded; + try { + Object parsed = Json.parse(body); + if(!(parsed instanceof Map)) { + return answer(400, "an export is a JSON object"); + } + // Counted on what was PARSED, before the sanitized copy is built: a body + // under the byte cap can still hold thousands of tiny spans, and + // validating and deep-copying all of them only to refuse the export + // spent the memory the span cap is there to bound. + if(OtlpSchema.countSpans((Map)parsed) > maxSpans) { + return answer(413, "export holds more than " + maxSpans + " spans"); + } + Map clean = OtlpSchema.sanitize((Map)parsed); + int spans = OtlpSchema.countSpans(clean); + if(spans == 0) { + return ok(); + } + encoded = exporter.isProtobuf() ? OtlpSchema.protobuf(clean) : OtlpSchema.json(clean); + } catch (Exception err) { + return answer(400, "not an OTLP trace export: " + err.getMessage()); + } + if(!exporter.addRelayed(encoded, + exporter.isProtobuf() ? "application/x-protobuf" : "application/json")) { + // OTLP/HTTP's own signal for "try again later", and what a client's + // exporter already backs off on. + return answer(503, "the relay queue is full"); + } + return ok(); + } + + private HttpServer.Response preflight() { + if(corsOrigin == null) { + return answer(405, "cross-origin export is not enabled on this relay"); + } + Map headers = cors(); + headers.put("Access-Control-Allow-Methods", "POST, OPTIONS"); + headers.put("Access-Control-Allow-Headers", "Content-Type, X-CN1-Telemetry-Token"); + headers.put("Access-Control-Max-Age", "86400"); + return HttpServer.Response.empty(204, "text/plain", headers); + } + + /** An empty ExportTraceServiceResponse: success, nothing rejected. */ + private HttpServer.Response ok() { + return new HttpServer.Response(200, "application/json", ascii("{}"), cors()); + } + + private HttpServer.Response answer(int status, String message) { + return new HttpServer.Response(status, "text/plain; charset=utf-8", ascii(message), cors()); + } + + private Map cors() { + Map headers = new LinkedHashMap(); + if(corsOrigin != null) { + headers.put("Access-Control-Allow-Origin", corsOrigin); + headers.put("Vary", "Origin"); + } + return headers; + } + + /** + * The UTF-8 length of {@code value}, counted without encoding it, and given up + * on as soon as it passes {@code limit}. An unpaired surrogate counts as the + * three bytes of the replacement character an encoder writes for it. + */ + static long utf8Length(String value, long limit) { + // Every character is at least one byte, so a long body is refused without + // walking it. + if(value.length() > limit) { + return value.length(); + } + long bytes = 0; + int n = value.length(); + for(int iter = 0 ; iter < n && bytes <= limit ; iter++) { + char c = value.charAt(iter); + if(c < 0x80) { + bytes++; + } else if(c < 0x800) { + bytes += 2; + } else if(Character.isHighSurrogate(c) && iter + 1 < n + && Character.isLowSurrogate(value.charAt(iter + 1))) { + bytes += 4; + iter++; + } else { + bytes += 3; + } + } + return bytes; + } + + private static byte[] utf8(String value) throws java.io.IOException { + return value.getBytes("UTF-8"); + } + + private static byte[] ascii(String value) { + byte[] out = new byte[value.length()]; + for(int iter = 0 ; iter < out.length ; iter++) { + char c = value.charAt(iter); + out[iter] = (byte)(c < 0x80 ? c : '?'); + } + return out; + } +} diff --git a/vm/backend/src/com/codename1/backend/otel/OtlpSchema.java b/vm/backend/src/com/codename1/backend/otel/OtlpSchema.java new file mode 100644 index 00000000000..a5695969896 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/otel/OtlpSchema.java @@ -0,0 +1,799 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.otel; + +import com.codename1.backend.ByteSink; +import com.codename1.backend.Json; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * The OTLP trace export request, as a table, and the protobuf encoder driven by + * it. + * + *

ONE MODEL, TWO ENCODINGS. An export is built as the tree OTLP/JSON + * describes -- Maps keyed by the lowerCamelCase field names, Lists for repeated + * fields -- and that tree is either written with {@link Json} or walked against + * this table to produce the binary protobuf form. The relay uses the same walk to + * re-encode what a client sent, which is also what validates it: a field this + * table does not name is dropped rather than forwarded, and a trace id that is + * not 32 hex digits is refused. + * + *

Protobuf because some collectors accept nothing else -- Dynatrace's OTLP + * endpoint is one -- and JSON because every collector accepts it and a person can + * read it. The field numbers are opentelemetry-proto's + * {@code opentelemetry/proto/trace/v1/trace.proto} and its common and resource + * messages; the round trip is checked against the generated protobuf classes in + * the module's tests. + */ +final class OtlpSchema { + // Field kinds. + private static final int STRING = 1; + /** A hex string in JSON, bytes on the wire. */ + private static final int HEX_BYTES = 2; + /** Base64 in JSON, bytes on the wire (AnyValue.bytesValue only). */ + private static final int BASE64_BYTES = 3; + private static final int FIXED64 = 4; + private static final int FIXED32 = 5; + /** uint32 counts and enums: a varint. */ + private static final int VARINT = 6; + private static final int INT64 = 7; + private static final int BOOL = 8; + private static final int DOUBLE = 9; + private static final int MESSAGE = 10; + + /** One field: its JSON name, its number, its kind, and for a message its type. */ + private static final class Field { + final String name; + final int number; + final int kind; + final boolean repeated; + Message type; + /** For HEX_BYTES: the exact length in bytes, or 0 for any. */ + final int bytes; + /** + * For HEX_BYTES: whether an all-zero value is refused. OTLP requires trace + * and span ids to be nonzero; a parent id is simply absent for a root. + */ + boolean nonZero; + /** + * For HEX_BYTES: an all-zero value is refused, but absent or empty is fine + * -- a parent id, which a root span leaves empty and no span may give as + * zeros. A zero one was forwarded after the relay answered 200. + */ + boolean nonZeroIfPresent; + + Field(String name, int number, int kind, boolean repeated, int bytes) { + this.name = name; + this.number = number; + this.kind = kind; + this.repeated = repeated; + this.bytes = bytes; + } + } + + private static final class Message { + final Field[] fields; + + Message(Field[] fields) { + this.fields = fields; + } + + Field field(String name) { + for(int iter = 0 ; iter < fields.length ; iter++) { + if(fields[iter].name.equals(name)) { + return fields[iter]; + } + } + return null; + } + } + + private static Field f(String name, int number, int kind) { + return new Field(name, number, kind, false, 0); + } + + private static Field hex(String name, int number, int length) { + return new Field(name, number, HEX_BYTES, false, length); + } + + /** A required id: a relay that forwarded a zero one would have answered 200 for a + * span the collector then rejects, with the app long past being able to retry. */ + private static Field id(String name, int number, int length) { + Field out = hex(name, number, length); + out.nonZero = true; + return out; + } + + /** An optional id: empty for none, never zeros when given. */ + private static Field optionalId(String name, int number, int length) { + Field out = hex(name, number, length); + out.nonZeroIfPresent = true; + return out; + } + + private static Field rep(String name, int number) { + return new Field(name, number, MESSAGE, true, 0); + } + + private static final Message ANY_VALUE; + private static final Message KEY_VALUE; + private static final Message ARRAY_VALUE; + private static final Message KEY_VALUE_LIST; + private static final Message RESOURCE; + private static final Message SCOPE; + private static final Message STATUS; + private static final Message EVENT; + private static final Message LINK; + private static final Message SPAN; + private static final Message SCOPE_SPANS; + private static final Message RESOURCE_SPANS; + /** ExportTraceServiceRequest, the body of POST /v1/traces. */ + private static final Message EXPORT; + + static { + Field arrayValue = f("arrayValue", 5, MESSAGE); + Field kvlistValue = f("kvlistValue", 6, MESSAGE); + ANY_VALUE = new Message(new Field[] { + f("stringValue", 1, STRING), + f("boolValue", 2, BOOL), + f("intValue", 3, INT64), + f("doubleValue", 4, DOUBLE), + arrayValue, + kvlistValue, + f("bytesValue", 7, BASE64_BYTES) + }); + Field kvValue = f("value", 2, MESSAGE); + kvValue.type = ANY_VALUE; + KEY_VALUE = new Message(new Field[] {f("key", 1, STRING), kvValue}); + Field arrayValues = rep("values", 1); + arrayValues.type = ANY_VALUE; + ARRAY_VALUE = new Message(new Field[] {arrayValues}); + Field listValues = rep("values", 1); + listValues.type = KEY_VALUE; + KEY_VALUE_LIST = new Message(new Field[] {listValues}); + arrayValue.type = ARRAY_VALUE; + kvlistValue.type = KEY_VALUE_LIST; + + RESOURCE = new Message(new Field[] { + attributes(1), f("droppedAttributesCount", 2, VARINT) + }); + SCOPE = new Message(new Field[] { + f("name", 1, STRING), f("version", 2, STRING), attributes(3), + f("droppedAttributesCount", 4, VARINT) + }); + STATUS = new Message(new Field[] {f("message", 2, STRING), f("code", 3, VARINT)}); + EVENT = new Message(new Field[] { + f("timeUnixNano", 1, FIXED64), f("name", 2, STRING), attributes(3), + f("droppedAttributesCount", 4, VARINT) + }); + LINK = new Message(new Field[] { + id("traceId", 1, 16), id("spanId", 2, 8), f("traceState", 3, STRING), + attributes(4), f("droppedAttributesCount", 5, VARINT), f("flags", 6, FIXED32) + }); + Field events = rep("events", 11); + events.type = EVENT; + Field links = rep("links", 13); + links.type = LINK; + Field status = f("status", 15, MESSAGE); + status.type = STATUS; + SPAN = new Message(new Field[] { + id("traceId", 1, 16), id("spanId", 2, 8), f("traceState", 3, STRING), + optionalId("parentSpanId", 4, 8), f("flags", 16, FIXED32), f("name", 5, STRING), + f("kind", 6, VARINT), f("startTimeUnixNano", 7, FIXED64), + f("endTimeUnixNano", 8, FIXED64), attributes(9), + f("droppedAttributesCount", 10, VARINT), events, + f("droppedEventsCount", 12, VARINT), links, + f("droppedLinksCount", 14, VARINT), status + }); + Field scope = f("scope", 1, MESSAGE); + scope.type = SCOPE; + Field spans = rep("spans", 2); + spans.type = SPAN; + SCOPE_SPANS = new Message(new Field[] {scope, spans, f("schemaUrl", 3, STRING)}); + Field resource = f("resource", 1, MESSAGE); + resource.type = RESOURCE; + Field scopeSpans = rep("scopeSpans", 2); + scopeSpans.type = SCOPE_SPANS; + RESOURCE_SPANS = new Message(new Field[] {resource, scopeSpans, f("schemaUrl", 3, STRING)}); + Field resourceSpans = rep("resourceSpans", 1); + resourceSpans.type = RESOURCE_SPANS; + EXPORT = new Message(new Field[] {resourceSpans}); + } + + private static Field attributes(int number) { + Field out = rep("attributes", number); + out.type = KEY_VALUE; + return out; + } + + private OtlpSchema() { + } + + /** The export request as OTLP/JSON. */ + static byte[] json(Map request) { + ByteSink out = new ByteSink(1024); + Json.write(request, out); + return copy(out); + } + + /** + * The export request as protobuf. + * + * @throws IOException when a value does not fit its field: a trace id of the + * wrong length, a count that is not a number. Only the relay can hit + * that, since it is the one caller whose tree came from outside. + */ + static byte[] protobuf(Map request) throws IOException { + ByteSink out = new ByteSink(1024); + writeMessage(EXPORT, request, out, 0); + return copy(out); + } + + /** + * Re-builds a tree, keeping only what this table names. The relay forwards the + * result instead of the client's own tree, so unknown fields -- whatever a + * client decided to attach -- never reach the collector. + */ + static Map sanitize(Map request) throws IOException { + return sanitizeMessage(EXPORT, request, 0); + } + + /** How many spans a request carries, for the relay's cap. */ + static int countSpans(Map request) { + int total = 0; + Object rs = request.get("resourceSpans"); + if(!(rs instanceof List)) { + return 0; + } + List resources = (List)rs; + for(int r = 0 ; r < resources.size() ; r++) { + Object resource = resources.get(r); + if(!(resource instanceof Map)) { + continue; + } + Object ss = ((Map)resource).get("scopeSpans"); + if(!(ss instanceof List)) { + continue; + } + List scopes = (List)ss; + for(int s = 0 ; s < scopes.size() ; s++) { + Object scope = scopes.get(s); + if(scope instanceof Map) { + Object spans = ((Map)scope).get("spans"); + if(spans instanceof List) { + total += ((List)spans).size(); + } + } + } + } + return total; + } + + /** AnyValue nests; nothing legitimate nests this deep. */ + private static final int MAX_DEPTH = 16; + + private static Map sanitizeMessage(Message type, Map value, int depth) throws IOException { + if(depth > MAX_DEPTH) { + throw new IOException("the export nests deeper than " + MAX_DEPTH + " levels"); + } + if(type == ANY_VALUE) { + // A oneof: exactly one alternative, or none. Keeping two forwarded an + // AnyValue a strict collector rejects -- after the relay had answered + // 200 -- and in protobuf the later tag silently replaced the earlier. + int present = 0; + for(int iter = 0 ; iter < type.fields.length ; iter++) { + if(value.get(type.fields[iter].name) != null) { + present++; + } + } + if(present > 1) { + throw new IOException("an AnyValue holds more than one of its alternatives"); + } + } + Map out = new java.util.LinkedHashMap(); + for(int iter = 0 ; iter < type.fields.length ; iter++) { + Field field = type.fields[iter]; + Object v = value.get(field.name); + if(v == null) { + if(field.nonZero) { + // A span or link with no id at all is as unusable as a zero one. + throw new IOException(field.name + " is required"); + } + continue; + } + if(field.kind != MESSAGE) { + // Checked by encoding it: the same rules, one place. + checkScalar(field, v); + out.put(field.name, v); + continue; + } + if(field.repeated) { + if(!(v instanceof List)) { + throw new IOException(field.name + " must be a list"); + } + List items = (List)v; + List kept = new java.util.ArrayList(items.size()); + for(int i = 0 ; i < items.size() ; i++) { + Object item = items.get(i); + if(!(item instanceof Map)) { + throw new IOException(field.name + " must hold objects"); + } + kept.add(sanitizeMessage(field.type, (Map)item, depth + 1)); + } + out.put(field.name, kept); + } else { + if(!(v instanceof Map)) { + throw new IOException(field.name + " must be an object"); + } + out.put(field.name, sanitizeMessage(field.type, (Map)v, depth + 1)); + } + } + return out; + } + + private static void checkScalar(Field field, Object value) throws IOException { + ByteSink scratch = new ByteSink(32); + writeScalar(field, value, scratch); + } + + private static void writeMessage(Message type, Map value, ByteSink out, int depth) + throws IOException { + if(depth > MAX_DEPTH) { + throw new IOException("the export nests deeper than " + MAX_DEPTH + " levels"); + } + for(int iter = 0 ; iter < type.fields.length ; iter++) { + Field field = type.fields[iter]; + Object v = value.get(field.name); + if(v == null) { + continue; + } + if(field.kind == MESSAGE) { + if(field.repeated) { + if(!(v instanceof List)) { + throw new IOException(field.name + " must be a list"); + } + List items = (List)v; + for(int i = 0 ; i < items.size() ; i++) { + Object item = items.get(i); + if(!(item instanceof Map)) { + throw new IOException(field.name + " must hold objects"); + } + writeNested(field, (Map)item, out, depth); + } + } else { + if(!(v instanceof Map)) { + throw new IOException(field.name + " must be an object"); + } + writeNested(field, (Map)v, out, depth); + } + continue; + } + writeScalar(field, v, out); + } + } + + /** Length-delimited: the child is encoded first so its length is known. */ + private static void writeNested(Field field, Map value, ByteSink out, int depth) + throws IOException { + ByteSink child = new ByteSink(64); + writeMessage(field.type, value, child, depth + 1); + tag(out, field.number, 2); + varint(out, child.length()); + out.put(child); + } + + private static void writeScalar(Field field, Object value, ByteSink out) throws IOException { + switch(field.kind) { + case STRING: { + if(!(value instanceof String)) { + throw new IOException(field.name + " must be a string"); + } + // The VM's native encoder, not ByteSink's loop: on the packaged + // runtime String.getBytes is the vectorized path. + byte[] text = ((String)value).getBytes("UTF-8"); + tag(out, field.number, 2); + varint(out, text.length); + out.put(text, 0, text.length); + return; + } + case HEX_BYTES: { + byte[] bytes = fromHex(field, value); + if(bytes.length == 0) { + // proto3 omits an empty bytes field; a root span's + // parentSpanId is written as "" in JSON and absent here. + return; + } + tag(out, field.number, 2); + varint(out, bytes.length); + out.put(bytes, 0, bytes.length); + return; + } + case BASE64_BYTES: { + if(!(value instanceof String)) { + throw new IOException(field.name + " must be a base64 string"); + } + byte[] bytes = fromBase64((String)value); + tag(out, field.number, 2); + varint(out, bytes.length); + out.put(bytes, 0, bytes.length); + return; + } + case FIXED64: { + long v = number(field, value); + // Every fixed64 in the trace schema is an unsigned nanosecond + // timestamp. A negative one was accepted, forwarded as an invalid + // value in JSON after the relay had answered 200, and in protobuf + // reinterpreted as a time in the far future. (The signed parse also + // stops at 2^63 ns, the year 2262, which no real span reaches.) + if(v < 0) { + throw new IOException(field.name + " must not be negative"); + } + tag(out, field.number, 1); + fixed64(out, v); + return; + } + case FIXED32: { + long v = number(field, value); + if(v < 0 || v > 0xffffffffL) { + throw new IOException(field.name + " does not fit 32 bits"); + } + tag(out, field.number, 5); + for(int i = 0 ; i < 4 ; i++) { + out.put((int)((v >>> (8 * i)) & 0xff)); + } + return; + } + case VARINT: { + long v = number(field, value); + if(v < 0 || v > 0xffffffffL) { + throw new IOException(field.name + " does not fit 32 bits"); + } + tag(out, field.number, 0); + varint(out, v); + return; + } + case INT64: { + long v = number(field, value); + tag(out, field.number, 0); + varint(out, v); + return; + } + case BOOL: { + if(!(value instanceof Boolean)) { + throw new IOException(field.name + " must be true or false"); + } + tag(out, field.number, 0); + out.put(((Boolean)value).booleanValue() ? 1 : 0); + return; + } + case DOUBLE: { + double d; + if(value instanceof Number) { + d = ((Number)value).doubleValue(); + } else if(value instanceof String) { + try { + d = Double.parseDouble((String)value); + } catch (NumberFormatException err) { + throw new IOException(field.name + " must be a number"); + } + } else { + throw new IOException(field.name + " must be a number"); + } + tag(out, field.number, 1); + fixed64(out, Double.doubleToLongBits(d)); + return; + } + default: + throw new IOException("unknown field kind for " + field.name); + } + } + + /** + * A 64-bit number from JSON. The proto3 JSON mapping writes 64-bit integers as + * STRINGS, because a JavaScript number loses precision past 2^53 and a + * nanosecond timestamp is well past it; a number is accepted too, as + * collectors do. + */ + private static long number(Field field, Object value) throws IOException { + if(value instanceof Long || value instanceof Integer || value instanceof Short + || value instanceof Byte) { + return ((Number)value).longValue(); + } + if(value instanceof Number) { + double d = ((Number)value).doubleValue(); + if(d != Math.floor(d) || Double.isInfinite(d) || Double.isNaN(d)) { + throw new IOException(field.name + " must be a whole number"); + } + // In range BEFORE the cast: Java saturates, so 1e100 became + // Long.MAX_VALUE and a different value than the client sent was + // forwarded -- or, as JSON, an out-of-range one a collector rejects. + if(!(d >= -9.223372036854775808E18 && d < 9.223372036854775808E18)) { + throw new IOException(field.name + " is outside the 64-bit range"); + } + return (long)d; + } + if(value instanceof String) { + // A signed 64-bit decimal, checked digit by digit. Accumulating + // without a check wrapped silently past 19 digits, so a value the + // client never sent was forwarded. Accumulated NEGATIVELY so the one + // value with no positive counterpart, Long.MIN_VALUE, still parses. + String text = (String)value; + boolean negative = text.startsWith("-"); + String digits = negative ? text.substring(1) : text; + if(digits.length() == 0 || digits.length() > 19) { + throw new IOException(field.name + " must be a whole number in the 64-bit range"); + } + for(int iter = 0 ; iter < digits.length() ; iter++) { + char c = digits.charAt(iter); + if(c < '0' || c > '9') { + throw new IOException(field.name + " must be a whole number"); + } + } + if(digits.length() == 19 && digits.compareTo( + negative ? "9223372036854775808" : "9223372036854775807") > 0) { + throw new IOException(field.name + " is outside the 64-bit range"); + } + long v = 0; + for(int iter = 0 ; iter < digits.length() ; iter++) { + v = v * 10 - (digits.charAt(iter) - '0'); + } + return negative ? v : -v; + } + throw new IOException(field.name + " must be a number"); + } + + private static byte[] fromHex(Field field, Object value) throws IOException { + if(!(value instanceof String)) { + throw new IOException(field.name + " must be a hex string"); + } + String text = (String)value; + if(text.length() == 0) { + if(field.nonZero) { + throw new IOException(field.name + " is required"); + } + return new byte[0]; + } + if(field.bytes > 0 && text.length() != field.bytes * 2) { + throw new IOException(field.name + " must be " + (field.bytes * 2) + " hex digits"); + } + byte[] out = new byte[text.length() / 2]; + if(out.length * 2 != text.length()) { + throw new IOException(field.name + " must be an even number of hex digits"); + } + for(int iter = 0 ; iter < out.length ; iter++) { + int hi = hexDigit(text.charAt(iter * 2)); + int lo = hexDigit(text.charAt(iter * 2 + 1)); + if(hi < 0 || lo < 0) { + throw new IOException(field.name + " must be hex digits"); + } + out[iter] = (byte)((hi << 4) | lo); + } + if(field.nonZero || field.nonZeroIfPresent) { + boolean zero = true; + for(int iter = 0 ; iter < out.length && zero ; iter++) { + zero = out[iter] == 0; + } + if(zero) { + throw new IOException(field.name + " must not be all zeros"); + } + } + return out; + } + + static int hexDigit(char c) { + if(c >= '0' && c <= '9') { + return c - '0'; + } + if(c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if(c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; + } + + /** + * Base64 as proto3's JSON mapping accepts it: the standard or URL-safe + * alphabet, padded or not. Everything is checked before a byte is produced -- + * no '=' except as final padding, no data after it, and no incomplete quartet + * (one lone character carries no whole byte). Stopping at the first '=' let + * "AA=garbage" through, and the relay then forwarded something other than + * what the client sent. + */ + private static byte[] fromBase64(String text) throws IOException { + int end = text.length(); + int padding = 0; + while(end > 0 && text.charAt(end - 1) == '=' && padding < 2) { + end--; + padding++; + } + if(padding > 0 && text.length() % 4 != 0) { + throw new IOException("bytesValue has misplaced padding"); + } + if(end % 4 == 1) { + throw new IOException("bytesValue is not whole base64"); + } + ByteSink out = new ByteSink(end); + int buffer = 0; + int bits = 0; + for(int iter = 0 ; iter < end ; iter++) { + char c = text.charAt(iter); + int v; + if(c >= 'A' && c <= 'Z') { + v = c - 'A'; + } else if(c >= 'a' && c <= 'z') { + v = c - 'a' + 26; + } else if(c >= '0' && c <= '9') { + v = c - '0' + 52; + } else if(c == '+' || c == '-') { + v = 62; + } else if(c == '/' || c == '_') { + v = 63; + } else { + throw new IOException("bytesValue must be base64"); + } + buffer = (buffer << 6) | v; + bits += 6; + if(bits >= 8) { + bits -= 8; + out.put((buffer >> bits) & 0xff); + } + } + return copy(out); + } + + private static void tag(ByteSink out, int number, int wireType) { + varint(out, ((long)number << 3) | wireType); + } + + /** Base-128, low group first. Negative int64 takes the full ten bytes, as protobuf does. */ + static void varint(ByteSink out, long value) { + while((value & ~0x7fL) != 0) { + out.put((int)((value & 0x7f) | 0x80)); + value >>>= 7; + } + out.put((int)value); + } + + private static void fixed64(ByteSink out, long value) { + for(int i = 0 ; i < 8 ; i++) { + out.put((int)((value >>> (8 * i)) & 0xff)); + } + } + + private static byte[] copy(ByteSink sink) { + byte[] out = new byte[sink.length()]; + System.arraycopy(sink.bytes(), 0, out, 0, out.length); + return out; + } + + /** + * ExportTraceServiceResponse's partial_success, as OTLP/JSON writes it: + * {@code {"partialSuccess":{"rejectedSpans":"3","errorMessage":"..."}}}. + */ + static void jsonPartialSuccess(String body, long[] rejected, String[] message) throws IOException { + Object parsed = Json.parse(body); + if(!(parsed instanceof Map)) { + return; + } + Object partial = ((Map)parsed).get("partialSuccess"); + if(!(partial instanceof Map)) { + return; + } + Object count = ((Map)partial).get("rejectedSpans"); + if(count != null) { + rejected[0] = number(f("rejectedSpans", 1, INT64), count); + } + Object text = ((Map)partial).get("errorMessage"); + if(text instanceof String) { + message[0] = (String)text; + } + } + + /** + * The same from the binary form: field 1 of the response is the + * ExportTracePartialSuccess message, whose field 1 is rejected_spans (int64) + * and field 2 error_message (string). Unknown fields are skipped, as protobuf + * requires, so a newer collector's response still reads. + */ + static void protobufPartialSuccess(byte[] body, long[] rejected, String[] message) + throws IOException { + int[] at = new int[1]; + while(at[0] < body.length) { + long key = readVarint(body, at); + int field = (int)(key >>> 3); + int wire = (int)(key & 7); + if(field == 1 && wire == 2) { + int length = (int)readVarint(body, at); + int end = at[0] + length; + if(length < 0 || end > body.length) { + throw new IOException("truncated response"); + } + while(at[0] < end) { + long inner = readVarint(body, at); + int innerField = (int)(inner >>> 3); + int innerWire = (int)(inner & 7); + if(innerField == 1 && innerWire == 0) { + rejected[0] = readVarint(body, at); + } else if(innerField == 2 && innerWire == 2) { + int n = (int)readVarint(body, at); + if(n < 0 || at[0] + n > end) { + throw new IOException("truncated response"); + } + message[0] = new String(body, at[0], n, "UTF-8"); + at[0] += n; + } else { + skip(body, at, innerWire); + } + } + } else { + skip(body, at, wire); + } + } + } + + private static long readVarint(byte[] data, int[] at) throws IOException { + long value = 0; + for(int shift = 0 ; shift < 64 ; shift += 7) { + if(at[0] >= data.length) { + throw new IOException("truncated varint"); + } + int b = data[at[0]++] & 0xff; + value |= (long)(b & 0x7f) << shift; + if((b & 0x80) == 0) { + return value; + } + } + throw new IOException("varint too long"); + } + + private static void skip(byte[] data, int[] at, int wire) throws IOException { + switch(wire) { + case 0: + readVarint(data, at); + return; + case 1: + at[0] += 8; + break; + case 2: + int n = (int)readVarint(data, at); + if(n < 0) { + throw new IOException("negative length"); + } + at[0] += n; + break; + case 5: + at[0] += 4; + break; + default: + throw new IOException("unsupported wire type " + wire); + } + if(at[0] > data.length) { + throw new IOException("truncated field"); + } + } +} diff --git a/vm/backend/src/com/codename1/backend/otel/OtlpTracer.java b/vm/backend/src/com/codename1/backend/otel/OtlpTracer.java new file mode 100644 index 00000000000..577c174d7b3 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/otel/OtlpTracer.java @@ -0,0 +1,1054 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.otel; + +import com.codename1.backend.Config; +import com.codename1.backend.Crypto; +import com.codename1.backend.HttpServer; +import com.codename1.backend.Span; +import com.codename1.backend.Tracer; +import com.codename1.backend.Tracing; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * OpenTelemetry tracing over OTLP/HTTP, with no OpenTelemetry library behind it. + * + *

The OpenTelemetry Java SDK cannot run here: the server is translated to C + * against a Java subset with no reflection and no service loading, and a binary + * that linked the SDK would be carrying it whether or not anyone traced. This is + * the part of it a server actually needs -- W3C trace context, the standard + * samplers, a bounded batch exporter, and the OTLP wire format in both its + * encodings -- and it is only in the binary when the project enables it. + * + *

Configured the way every OpenTelemetry SDK is, so operations tooling works + * unchanged. Each setting has a {@code cn1.otel.*} key, readable from + * application.properties like any other, and the standard {@code OTEL_*} + * environment variable is honoured for it too: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
keyvariabledefault
cn1.otel.endpointOTEL_EXPORTER_OTLP_ENDPOINThttp://localhost:4318; /v1/traces is appended
cn1.otel.traces.endpointOTEL_EXPORTER_OTLP_TRACES_ENDPOINTused as it is, and wins over the one above
cn1.otel.headersOTEL_EXPORTER_OTLP_HEADERSnone; {@code name=value,name=value}
cn1.otel.protocolOTEL_EXPORTER_OTLP_PROTOCOLhttp/protobuf, or http/json
cn1.otel.service.nameOTEL_SERVICE_NAMEwhat the build named it, else unknown_service
cn1.otel.resource.attributesOTEL_RESOURCE_ATTRIBUTESnone
cn1.otel.samplerOTEL_TRACES_SAMPLERparentbased_always_on
cn1.otel.sampler.argOTEL_TRACES_SAMPLER_ARGthe ratio, 1
cn1.otel.disabledOTEL_SDK_DISABLEDfalse
+ * + *

{@code cn1.otel.attributes.exclude} names attributes never to record, as a + * comma separated list ({@code db.query.text,user_agent.original}), and + * {@code cn1.otel.relay=true} opens the endpoint the app's own spans are relayed + * through; see {@link OtlpRelay}. + */ +public final class OtlpTracer implements Tracer { + public static final String DISABLED = "cn1.otel.disabled"; + public static final String ENDPOINT = "cn1.otel.endpoint"; + public static final String TRACES_ENDPOINT = "cn1.otel.traces.endpoint"; + public static final String HEADERS = "cn1.otel.headers"; + public static final String TRACES_HEADERS = "cn1.otel.traces.headers"; + public static final String PROTOCOL = "cn1.otel.protocol"; + public static final String TRACES_PROTOCOL = "cn1.otel.traces.protocol"; + public static final String SERVICE_NAME = "cn1.otel.service.name"; + public static final String RESOURCE_ATTRIBUTES = "cn1.otel.resource.attributes"; + public static final String SAMPLER = "cn1.otel.sampler"; + public static final String SAMPLER_ARG = "cn1.otel.sampler.arg"; + public static final String ATTRIBUTES_EXCLUDE = "cn1.otel.attributes.exclude"; + public static final String QUEUE_SIZE = "cn1.otel.queue.size"; + public static final String BATCH_SIZE = "cn1.otel.batch.size"; + public static final String EXPORT_DELAY = "cn1.otel.export.delayMillis"; + public static final String RELAY = "cn1.otel.relay"; + public static final String RELAY_PATH = "cn1.otel.relay.path"; + public static final String RELAY_TOKEN = "cn1.otel.relay.token"; + public static final String RELAY_MAX_BYTES = "cn1.otel.relay.maxBytes"; + public static final String RELAY_MAX_SPANS = "cn1.otel.relay.maxSpans"; + public static final String RELAY_CORS_ORIGIN = "cn1.otel.relay.corsOrigin"; + + /** What the instrumentation scope is called in every export. */ + static final String SCOPE_NAME = "com.codename1.backend"; + + private final String defaultServiceName; + private Sampler sampler; + private Set excluded = new HashSet(); + private BatchExporter exporter; + private OtlpRelay relay; + /** Ids fetched per call into the secure generator; see seedIds. */ + private static final int ID_BLOCK = 64; + private final long[] ids = new long[ID_BLOCK]; + private int idsLeft; + private byte[] idKey; + private long idCounter; + + /** A tracer whose service name comes from configuration alone. */ + public OtlpTracer() { + this(null); + } + + /** + * @param defaultServiceName used when neither {@code cn1.otel.service.name} + * nor {@code OTEL_SERVICE_NAME} is set; the build passes the name + * {@code @OpenTelemetry} gave + */ + public OtlpTracer(String defaultServiceName) { + this.defaultServiceName = defaultServiceName; + } + + /** + * A tracer opened against {@code config}, or null when the configuration + * turns tracing off. For a program that starts {@link HttpServer} or the + * Lambda loop itself: + * + * 

+     *   Tracing.install(OtlpTracer.open(Config.load(), "orders"));
+     *
+ */ + public static OtlpTracer open(Config config, String defaultServiceName) throws IOException { + OtlpTracer tracer = new OtlpTracer(defaultServiceName); + return tracer.open(config) ? tracer : null; + } + + public boolean open(Config config) throws IOException { + if(config.getBoolean(DISABLED, false)) { + return false; + } + sampler = Sampler.parse(config.get(SAMPLER), config.get(SAMPLER_ARG)); + excluded = splitSet(config.get(ATTRIBUTES_EXCLUDE)); + + String protocol = config.get(TRACES_PROTOCOL, config.get(PROTOCOL, "http/protobuf")).trim(); + boolean protobuf; + if("http/protobuf".equals(protocol)) { + protobuf = true; + } else if("http/json".equals(protocol)) { + protobuf = false; + } else { + // grpc is the one other value the specification names, and the one a + // copied collector config most often carries. Refused by name: the + // server would otherwise start and send nothing anyone receives. + throw new IOException(PROTOCOL + " is '" + protocol + "'; this server exports " + + "OTLP over HTTP, so use http/protobuf or http/json, and point the " + + "endpoint at the collector's HTTP port (4318 by default, not 4317)"); + } + + String endpoint = config.get(TRACES_ENDPOINT); + if(endpoint == null || endpoint.trim().length() == 0) { + String base = config.get(ENDPOINT, "http://localhost:4318").trim(); + endpoint = appendTracesPath(base); + } + endpoint = endpoint.trim(); + if(!hasHttpAuthority(endpoint)) { + throw new IOException("The trace endpoint must be an http or https URL naming " + + "a host and is '" + BatchExporter.redact(endpoint) + "'"); + } + + // The signal-specific setting REPLACES the generic one, as the endpoint and + // protocol settings do and as the specification says: it is not a list to + // append to. Appending sent both, and Web sends every line, so a deployment + // that set a traces-only token next to a generic one exported two + // Authorization headers and was refused by collectors that reject that. + List headers = new ArrayList(); + String tracesHeaders = config.get(TRACES_HEADERS); + parseHeaders(tracesHeaders != null ? tracesHeaders : config.get(HEADERS), headers); + + Map resourceAttributes = new LinkedHashMap(); + parsePairs(config.get(RESOURCE_ATTRIBUTES), resourceAttributes, RESOURCE_ATTRIBUTES); + // Each source in turn, a blank one counting as absent: tested raw, a name + // of " " was chosen and then trimmed to an empty service.name, where the + // specification wants unknown_service. + Object fromResource = resourceAttributes.get("service.name"); + String service = nonBlank(config.get(SERVICE_NAME)); + if(service == null) { + service = nonBlank(fromResource == null ? null : String.valueOf(fromResource)); + } + if(service == null) { + service = nonBlank(defaultServiceName); + } + resourceAttributes.put("service.name", service == null ? "unknown_service" : service); + resourceAttributes.put("telemetry.sdk.name", "codenameone"); + resourceAttributes.put("telemetry.sdk.language", "java"); + Map resource = new LinkedHashMap(); + resource.put("attributes", keyValues(resourceAttributes)); + + int queue = positive(config, QUEUE_SIZE, 2048); + int batch = Math.min(queue, positive(config, BATCH_SIZE, 512)); + int delay = positive(config, EXPORT_DELAY, 5000); + int relayBytes = positive(config, RELAY_MAX_BYTES, 1024 * 1024); + + seedIds(); + exporter = new BatchExporter(endpoint, headers, protobuf, resource, queue, batch, + delay, relayBytes * 4L); + if(config.getBoolean(RELAY, false)) { + String path = canonicalPath(config.get(RELAY_PATH, "/otel/v1/traces").trim()); + if(path == null) { + throw new IOException(RELAY_PATH + " must be a path such as /otel/v1/traces: " + + "it starts with /, has no query or fragment, and uses only the " + + "ASCII characters a URL path allows (percent-encode anything " + + "else); it is '" + path + "'"); + } + String token = config.get(RELAY_TOKEN); + if(token != null && token.length() > 0 && !sendableFieldValue(token)) { + // Refused here because no client could ever present it: the request + // parser refuses a control character in a header value and trims + // surrounding spaces and tabs, so a trailing newline from a mounted + // secret made every relay export a 401 the app drops silently. The + // value itself stays out of the message -- it is a secret. + throw new IOException(RELAY_TOKEN + " holds a control character or " + + "leading or trailing whitespace, so no request can carry it " + + "in a header; remove it (a secret file's trailing newline is " + + "the usual cause)"); + } + relay = new OtlpRelay(path, token, relayBytes, + positive(config, RELAY_MAX_SPANS, 1000), corsOrigin(config), + exporter); + } + exporter.start(); + return true; + } + + /** + * Whether a request header could carry this value exactly: no control + * character but tab, and no space or tab at either end, which the request + * parser trims as surrounding whitespace. + */ + static boolean sendableFieldValue(String value) { + int last = value.length() - 1; + for(int iter = 0 ; iter <= last ; iter++) { + char c = value.charAt(iter); + if((c < 0x20 && c != '\t') || c == 0x7f) { + return false; + } + if((iter == 0 || iter == last) && (c == ' ' || c == '\t')) { + return false; + } + } + return true; + } + + public Span startSpan(String name, int kind, Span parent, String traceparent, + String tracestate) { + long hi; + long lo; + long parentId = 0; + boolean hasParent = false; + boolean parentSampled = false; + boolean remote = false; + String state = null; + // Only a parent THIS tracer made. After Tracing.install() replaces the + // tracer, a request still in flight holds the old one's span as current, + // and adopting it filed the new tracer's children under the old trace, its + // sampling decision and its clock -- exported, possibly, to another + // collector than their parent's. Such a child starts a trace of its own. + OtelSpan local = parent instanceof OtelSpan && ((OtelSpan)parent).isFrom(this) + ? (OtelSpan)parent : null; + if(local != null) { + hi = local.traceHi; + lo = local.traceLo; + parentId = local.spanId; + parentSampled = local.sampled; + state = local.tracestate; + hasParent = true; + } else { + TraceContext context = TraceContext.parse(traceparent); + if(context != null) { + hi = context.traceHi; + lo = context.traceLo; + parentId = context.spanId; + parentSampled = context.sampled(); + state = TraceContext.vetTracestate(tracestate); + hasParent = true; + remote = true; + } else { + hi = nextId(); + lo = nextId(); + } + } + boolean sampled = sampler.sample(hasParent, parentSampled, lo); + return new OtelSpan(this, name, kind, hi, lo, nextId(), parentId, remote, sampled, state, + local); + } + + public void flush(int timeoutMillis) { + if(exporter != null) { + exporter.flush(timeoutMillis); + } + } + + public void shutdown(int timeoutMillis) { + if(exporter != null) { + exporter.shutdown(timeoutMillis); + } + } + + public HttpServer.Handler relay() { + return relay; + } + + public void metrics(Map out) { + if(exporter != null) { + exporter.metrics(out); + } + } + + void ended(OtelSpan span) { + exporter.add(span); + } + + /** + * {@code path} as the server will compare it, or null when it is not an + * origin-form path the relay could ever match. + * + *

Only RFC 3986 pchar and "/": a non-ASCII character used to be folded to + * '?', so the relay listened somewhere nobody configured, and a query or + * fragment could never match, since only the path is compared. + * + *

Then normalized as the server normalizes every request path before + * {@code Request.pathIs} compares it (RFC 3986 6.2.2): an escaped unreserved + * character is decoded and a kept escape gets upper-case hex digits. Compared + * as configured, {@code /otel/%74races} or {@code /otel/%2f} could never + * match any request. A '%' not followed by two hex digits is refused. + */ + static String canonicalPath(String path) { + if(path.length() == 0 || path.charAt(0) != '/') { + return null; + } + StringBuilder out = new StringBuilder(path.length()); + for(int iter = 0 ; iter < path.length() ; iter++) { + char c = path.charAt(iter); + if(c == '%') { + int hi = iter + 2 < path.length() ? hexValue(path.charAt(iter + 1)) : -1; + int lo = hi < 0 ? -1 : hexValue(path.charAt(iter + 2)); + if(lo < 0) { + return null; + } + char decoded = (char)((hi << 4) | lo); + if(isUnreserved(decoded)) { + out.append(decoded); + } else { + out.append('%').append(Character.toUpperCase(path.charAt(iter + 1))) + .append(Character.toUpperCase(path.charAt(iter + 2))); + } + iter += 2; + continue; + } + if(!isUnreserved(c) && "!$&'()*+,;=:@/".indexOf(c) < 0) { + return null; + } + out.append(c); + } + return out.toString(); + } + + private static boolean isUnreserved(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') + || c == '-' || c == '.' || c == '_' || c == '~'; + } + + private static int hexValue(char c) { + if(c >= '0' && c <= '9') { + return c - '0'; + } + if(c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if(c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; + } + + /** + * {@code cn1.otel.relay.corsOrigin}, checked: {@code *}, or ONE serialized + * origin -- scheme, host and port, nothing after. The value goes verbatim into + * Access-Control-Allow-Origin, and a browser matches it character for + * character against the page's origin, so a path, a trailing slash or a + * comma-separated list matches nothing: every preflight failed while the + * backend and the app's fail-silent exporter both looked configured. + */ + static String corsOrigin(Config config) throws IOException { + String value = config.get(RELAY_CORS_ORIGIN); + if(value == null || value.trim().length() == 0) { + return null; + } + value = value.trim(); + if("*".equals(value)) { + return value; + } + int start = value.regionMatches(true, 0, "https://", 0, 8) ? 8 + : value.regionMatches(true, 0, "http://", 0, 7) ? 7 : -1; + boolean valid = start > 0 && hasHttpAuthority(value) + && value.indexOf('/', start) < 0 && value.indexOf('?') < 0 + && value.indexOf('#') < 0 && value.indexOf('@') < 0 + && value.indexOf(',') < 0; + if(!valid) { + // Redacted: a value refused for its userinfo, query or fragment is + // refused BECAUSE it carries one, and this message goes to deploy logs. + throw new IOException(RELAY_CORS_ORIGIN + " must be * or one origin such as " + + "https://app.example.com (scheme, host and port, with no path or " + + "trailing slash); it is '" + BatchExporter.redact(value) + "'"); + } + return serializedOrigin(value, start); + } + + /** + * An origin as a browser serializes it -- lower-case scheme and host, no + * default port -- since that is the string Access-Control-Allow-Origin is + * compared against, character for character. HTTPS://APP.EXAMPLE.COM:443 was + * accepted as written and matched no page. Called on a value already checked + * to be scheme://host[:port]. + */ + private static String serializedOrigin(String value, int start) { + String scheme = start == 8 ? "https" : "http"; + String hostPort = value.substring(start); + int close = hostPort.lastIndexOf(']'); + int colon = hostPort.lastIndexOf(':'); + String host = hostPort; + String port = null; + if(colon > close) { + host = hostPort.substring(0, colon); + port = hostPort.substring(colon + 1); + } + StringBuilder out = new StringBuilder(value.length()); + out.append(scheme).append("://"); + for(int iter = 0 ; iter < host.length() ; iter++) { + char c = host.charAt(iter); + out.append(c >= 'A' && c <= 'Z' ? (char)(c + 32) : c); + } + if(port != null && port.length() > 0) { + int number = Integer.parseInt(port); + if(number != (start == 8 ? 443 : 80)) { + out.append(':').append(number); + } + } + return out.toString(); + } + + /** + * Whether {@code url} is http or https with a host, and a valid port if it + * names one. A bare {@code https://} passed a scheme check and started an + * exporter whose every POST then failed: the server ran, validated, and + * produced no traces. + */ + static boolean hasHttpAuthority(String url) { + // No fragment. HTTP never sends one, so a credential kept there never + // reached the collector, and every export was refused silently. + if(url.indexOf('#') >= 0) { + return false; + } + // The WHOLE URL first, by the rule Web applies when it sends: no space, + // control or DEL anywhere. Only the authority was checked, so a space in + // the path passed and every export then failed at transport, silently. + for(int iter = 0 ; iter < url.length() ; iter++) { + char c = url.charAt(iter); + if(c <= 0x20 || c == 0x7f) { + return false; + } + } + int start; + if(url.regionMatches(true, 0, "http://", 0, 7)) { + start = 7; + } else if(url.regionMatches(true, 0, "https://", 0, 8)) { + start = 8; + } else { + return false; + } + int end = url.length(); + for(int iter = start ; iter < url.length() ; iter++) { + char c = url.charAt(iter); + if(c == '/' || c == '?' || c == '#') { + end = iter; + break; + } + } + String authority = url.substring(start, end); + int at = authority.lastIndexOf('@'); + if(at >= 0 && !validUserinfo(authority.substring(0, at))) { + return false; + } + String hostPort = at < 0 ? authority : authority.substring(at + 1); + String host; + String port = null; + if(hostPort.startsWith("[")) { + int close = hostPort.indexOf(']'); + if(close < 0) { + return false; + } + host = hostPort.substring(1, close); + // An IPv6 literal: hex digits and colons, with dots for an embedded + // IPv4 tail. Anything else is not an address any stack will parse. + if(!isIpv6(host)) { + return false; + } + String rest = hostPort.substring(close + 1); + if(rest.length() > 0) { + if(rest.charAt(0) != ':') { + return false; + } + port = rest.substring(1); + } + } else { + int colon = hostPort.lastIndexOf(':'); + host = colon < 0 ? hostPort : hostPort.substring(0, colon); + port = colon < 0 ? null : hostPort.substring(colon + 1); + // A DNS name or an IPv4 address: letters, digits, '-', '.', and the + // other unreserved characters. A space, a control, a backslash or a + // stray bracket passed the emptiness check and started an exporter no + // resolver or libcurl could connect with, so every span was lost. + if(!onlyChars(host, "-._~")) { + return false; + } + } + if(host.length() == 0) { + return false; + } + // An empty port ("host:") is the scheme's default, as RFC 3986 allows. + if(port != null && port.length() > 0) { + if(port.length() > 5) { + return false; + } + int value = 0; + for(int iter = 0 ; iter < port.length() ; iter++) { + char c = port.charAt(iter); + if(c < '0' || c > '9') { + return false; + } + value = value * 10 + (c - '0'); + } + return value > 0 && value <= 65535; + } + return true; + } + + // An IPv6 address by its STRUCTURE (RFC 4291 2.2), not its characters: groups + // of one to four hex digits, at most one "::", eight groups without it and at + // most seven with it, and optionally a dotted IPv4 tail counting as two. A + // character check let "[:::]" through, and the transport refused every export. + static boolean isIpv6(String s) { + int n = s.length(); + if (n == 0) { + return false; + } + int groups = 0; + boolean compressed = false; + int i = 0; + if (s.startsWith("::")) { + compressed = true; + i = 2; + if (i == n) { + return true; + } + } else if (s.charAt(0) == ':') { + return false; + } + while (i < n) { + int j = i; + while (j < n && s.charAt(j) != ':') { + j++; + } + String part = s.substring(i, j); + if (part.indexOf('.') >= 0) { + if (j != n || !isIpv4(part)) { + return false; + } + groups += 2; + } else { + if (part.length() < 1 || part.length() > 4) { + return false; + } + for (int k = 0; k < part.length(); k++) { + char c = part.charAt(k); + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) { + return false; + } + } + groups++; + } + if (j == n) { + break; + } + if (j + 1 < n && s.charAt(j + 1) == ':') { + if (compressed) { + return false; + } + compressed = true; + i = j + 2; + } else { + i = j + 1; + if (i == n) { + return false; + } + } + } + return compressed ? groups <= 7 : groups == 8; + } + + // Four decimal parts, each 0 to 255. + static boolean isIpv4(String s) { + int parts = 0; + int i = 0; + while (i <= s.length()) { + int j = s.indexOf('.', i); + if (j < 0) { + j = s.length(); + } + String part = s.substring(i, j); + if (part.length() < 1 || part.length() > 3) { + return false; + } + int value = 0; + for (int k = 0; k < part.length(); k++) { + char c = part.charAt(k); + if (c < '0' || c > '9') { + return false; + } + value = value * 10 + (c - '0'); + } + if (value > 255) { + return false; + } + parts++; + i = j + 1; + } + return parts == 4; + } + + /** {@code value} trimmed, or null when that leaves nothing. */ + private static String nonBlank(String value) { + if(value == null) { + return null; + } + String trimmed = value.trim(); + return trimmed.length() == 0 ? null : trimmed; + } + + /** + * RFC 3986 userinfo: unreserved characters, sub-delims, ':' and complete + * percent escapes. Skipped over, a space, a control or a stray '%' in it + * passed validation and failed only at transport, where exports fail silently. + */ + static boolean validUserinfo(String userinfo) { + for(int iter = 0 ; iter < userinfo.length() ; iter++) { + char c = userinfo.charAt(iter); + if(c == '%') { + if(iter + 2 >= userinfo.length() || OtlpSchema.hexDigit(userinfo.charAt(iter + 1)) < 0 + || OtlpSchema.hexDigit(userinfo.charAt(iter + 2)) < 0) { + return false; + } + iter += 2; + continue; + } + if(!onlyChars(String.valueOf(c), "-._~!$&'()*+,;=:")) { + return false; + } + } + return true; + } + + /** Whether every character of {@code value} is an ASCII letter, a digit, or one of {@code extra}. */ + private static boolean onlyChars(String value, String extra) { + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + if(!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') + || extra.indexOf(c) >= 0)) { + return false; + } + } + return true; + } + + boolean excluded(String key) { + return !excluded.isEmpty() && excluded.contains(key); + } + + // ------------------------------------------------------------------ + // Ids + // ------------------------------------------------------------------ + + /** + * Ids come straight from the platform's secure generator, fetched a block at a + * time so the native call is paid once per ID_BLOCK ids rather than per span. + * + * They used to be the output of SplitMix64 over a securely seeded counter. That + * is a bijection with a public inverse: one span id seen in a response header or + * a log recovers the state, and from it every id this process issues next -- + * other users' trace ids included, which is exactly what makes a trace id worth + * guessing (joining someone else's trace, or predicting a correlation id another + * system trusts). + * + * A refill that fails after open() succeeded falls back to HMAC-SHA256 under a + * key drawn from the same generator at open() and never emitted, over a counter. + * That is still unpredictable without the key; it exists because nextId() runs + * on the request path, where throwing would fail the request being traced. + * open() itself fails when there is no secure generator at all. + */ + private void seedIds() throws IOException { + byte[] key = Crypto.randomBytes(32); + synchronized(this) { + idKey = key; + idsLeft = 0; + } + } + + private synchronized long nextId() { + long z; + do { + if(idsLeft == 0) { + refillIds(); + } + z = ids[--idsLeft]; + } while(z == 0); + return z; + } + + private void refillIds() { + byte[] block; + try { + block = Crypto.randomBytes(ID_BLOCK * 8); + } catch(IOException err) { + block = null; + } + if(block == null || block.length < ID_BLOCK * 8) { + block = new byte[ID_BLOCK * 8]; + byte[] counter = new byte[8]; + for(int off = 0 ; off < block.length ; off += 32) { + long c = ++idCounter; + for(int b = 0 ; b < 8 ; b++) { + counter[b] = (byte)(c >>> (56 - 8 * b)); + } + byte[] mac = Crypto.hmacSha256(idKey, counter); + if(mac == null || mac.length < 32) { + throw new IllegalStateException("No secure randomness for trace ids"); + } + System.arraycopy(mac, 0, block, off, Math.min(32, block.length - off)); + } + } + for(int iter = 0 ; iter < ID_BLOCK ; iter++) { + long v = 0; + for(int b = 0 ; b < 8 ; b++) { + v = (v << 8) | (block[iter * 8 + b] & 0xff); + } + ids[iter] = v; + } + idsLeft = ID_BLOCK; + } + + // ------------------------------------------------------------------ + // The export tree + // ------------------------------------------------------------------ + + /** An ExportTraceServiceRequest for these spans, as OTLP/JSON's tree. */ + static Map exportRequest(Map resource, List spans) { + List encoded = new ArrayList(spans.size()); + for(int iter = 0 ; iter < spans.size() ; iter++) { + Object span = spans.get(iter); + if(span instanceof OtelSpan) { + encoded.add(spanTree((OtelSpan)span)); + } + } + Map scope = new LinkedHashMap(); + scope.put("name", SCOPE_NAME); + Map scopeSpans = new LinkedHashMap(); + scopeSpans.put("scope", scope); + scopeSpans.put("spans", encoded); + List scopes = new ArrayList(1); + scopes.add(scopeSpans); + Map resourceSpans = new LinkedHashMap(); + resourceSpans.put("resource", resource); + resourceSpans.put("scopeSpans", scopes); + List all = new ArrayList(1); + all.add(resourceSpans); + Map request = new LinkedHashMap(); + request.put("resourceSpans", all); + return request; + } + + private static Map spanTree(OtelSpan span) { + Map out = new LinkedHashMap(); + out.put("traceId", TraceContext.hex(span.traceHi) + TraceContext.hex(span.traceLo)); + out.put("spanId", TraceContext.hex(span.spanId)); + if(span.tracestate != null) { + out.put("traceState", span.tracestate); + } + if(span.parentId != 0) { + out.put("parentSpanId", TraceContext.hex(span.parentId)); + } + // Trace flags in the low byte, then HAS_IS_REMOTE and IS_REMOTE: whether + // the parent was in another process, which a backend uses to draw the + // service boundary. Only when there IS a parent: the bits describe the + // parent's context (trace.proto: "unknown, is not remote, is remote"), and + // setting HAS_IS_REMOTE on a root claimed a local parent it does not have. + long flags = span.sampled ? 1 : 0; + if(span.parentId != 0) { + flags |= 0x100 | (span.parentRemote ? 0x200 : 0); + } + out.put("flags", Long.valueOf(flags)); + out.put("name", span.name); + out.put("kind", Integer.valueOf(span.kind)); + // Strings, as proto3's JSON mapping writes 64-bit integers. + out.put("startTimeUnixNano", String.valueOf(span.startEpochNanos)); + out.put("endTimeUnixNano", String.valueOf(span.endEpochNanos)); + out.put("attributes", keyValues(span.attributes)); + if(span.droppedAttributes > 0) { + out.put("droppedAttributesCount", Integer.valueOf(span.droppedAttributes)); + } + if(span.events != null && !span.events.isEmpty()) { + List events = new ArrayList(span.events.size()); + for(int iter = 0 ; iter < span.events.size() ; iter++) { + Object[] event = (Object[])span.events.get(iter); + Map e = new LinkedHashMap(); + e.put("timeUnixNano", String.valueOf(event[0])); + e.put("name", event[1]); + e.put("attributes", keyValues((Map)event[2])); + events.add(e); + } + out.put("events", events); + } + if(span.droppedEvents > 0) { + out.put("droppedEventsCount", Integer.valueOf(span.droppedEvents)); + } + if(span.statusCode != 0) { + Map status = new LinkedHashMap(); + if(span.statusMessage != null) { + status.put("message", span.statusMessage); + } + status.put("code", Integer.valueOf(span.statusCode)); + out.put("status", status); + } + return out; + } + + static List keyValues(Map attributes) { + List out = new ArrayList(attributes == null ? 0 : attributes.size()); + if(attributes == null) { + return out; + } + Iterator it = attributes.entrySet().iterator(); + while(it.hasNext()) { + Map.Entry entry = (Map.Entry)it.next(); + Object v = entry.getValue(); + Map value = new LinkedHashMap(); + if(v instanceof Boolean) { + value.put("boolValue", v); + } else if(v instanceof Long || v instanceof Integer) { + value.put("intValue", String.valueOf(v)); + } else if(v instanceof Double) { + value.put("doubleValue", v); + } else { + value.put("stringValue", String.valueOf(v)); + } + Map kv = new LinkedHashMap(); + kv.put("key", String.valueOf(entry.getKey())); + kv.put("value", value); + out.add(kv); + } + return out; + } + + // ------------------------------------------------------------------ + // Configuration parsing + // ------------------------------------------------------------------ + + /** + * The generic endpoint plus /v1/traces, appended to the PATH. An endpoint can + * carry its credential in the query ({@code https://c.example/otlp?api-key=...}); + * appending to the whole string put the path inside the key's value and sent + * the export to the base path with a corrupted credential. + */ + static String appendTracesPath(String base) { + int cut = base.length(); + int query = base.indexOf('?'); + int fragment = base.indexOf('#'); + if(query >= 0) { + cut = query; + } + if(fragment >= 0 && fragment < cut) { + cut = fragment; + } + String path = base.substring(0, cut); + while(path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + } + return path + "/v1/traces" + base.substring(cut); + } + + private static int positive(Config config, String key, int fallback) throws IOException { + int value = config.getInt(key, fallback); + if(value <= 0) { + throw new IOException(key + " must be a positive number and is " + value); + } + return value; + } + + private static Set splitSet(String list) { + Set out = new HashSet(); + if(list == null) { + return out; + } + int at = 0; + while(at <= list.length()) { + int comma = list.indexOf(',', at); + if(comma < 0) { + comma = list.length(); + } + String item = list.substring(at, comma).trim(); + if(item.length() > 0) { + out.add(item); + } + at = comma + 1; + } + return out; + } + + /** OTEL_EXPORTER_OTLP_HEADERS: {@code name=value,...}, values percent-encoded. */ + private static void parseHeaders(String text, List out) throws IOException { + Map pairs = new LinkedHashMap(); + parsePairs(text, pairs, HEADERS); + Iterator it = pairs.entrySet().iterator(); + while(it.hasNext()) { + Map.Entry entry = (Map.Entry)it.next(); + String name = String.valueOf(entry.getKey()); + if(name.length() == 0) { + throw new IOException(HEADERS + " has an entry with no header name"); + } + if(name.equalsIgnoreCase("content-type")) { + // The exporter sets it from the protocol, and Web sends every line, + // so a configured one went out as a SECOND Content-Type -- one of + // which contradicts the body. + throw new IOException(HEADERS + " sets Content-Type, which the exporter " + + "sets from " + PROTOCOL); + } + for(int iter = 0 ; iter < name.length() ; iter++) { + char c = name.charAt(iter); + boolean token = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') || "!#$%&'*+-.^_`|~".indexOf(c) >= 0; + if(!token) { + throw new IOException(HEADERS + " names a header '" + name + + "' that is not a valid header name"); + } + } + out.add(name + ": " + entry.getValue()); + } + // A value is percent-decoded, so %0A becomes a real newline here. Checked + // now, by the rules Web applies when it sends: accepting it started a + // server that then failed every export on that check and dropped every + // span. The message names the header, never the value -- it is usually a + // credential. + try { + Tracing.checkHeaderLines(out); + } catch (IOException err) { + IOException refused = new IOException(HEADERS + ": " + err.getMessage()); + refused.initCause(err); + throw refused; + } + } + + /** + * The W3C Baggage-style list both OTEL_EXPORTER_OTLP_HEADERS and + * OTEL_RESOURCE_ATTRIBUTES use. A malformed entry is an error, not skipped: + * the usual one is an Authorization header whose value was pasted with a + * space where the specification wants %20, and silently dropping it is a + * collector that answers 401 to every export. + */ + static void parsePairs(String text, Map out, String key) throws IOException { + if(text == null) { + return; + } + int at = 0; + while(at < text.length()) { + int comma = text.indexOf(',', at); + if(comma < 0) { + comma = text.length(); + } + String item = text.substring(at, comma).trim(); + at = comma + 1; + if(item.length() == 0) { + continue; + } + int eq = item.indexOf('='); + if(eq <= 0) { + throw new IOException(key + " must be name=value pairs separated by commas"); + } + out.put(item.substring(0, eq).trim(), percentDecode(item.substring(eq + 1).trim(), key)); + } + } + + private static String percentDecode(String value, String key) throws IOException { + if(value.indexOf('%') < 0) { + return value; + } + com.codename1.backend.ByteSink bytes = new com.codename1.backend.ByteSink(value.length()); + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + if(c != '%') { + // By CODE POINT: a supplementary character is two chars here, and + // encoding each half on its own wrote invalid UTF-8 that the String + // constructor below replaced -- corrupting a resource value, or a + // credential so that every export was refused. + if(Character.isHighSurrogate(c) && iter + 1 < value.length() + && Character.isLowSurrogate(value.charAt(iter + 1))) { + bytes.putCodePoint(Character.toCodePoint(c, value.charAt(iter + 1))); + iter++; + continue; + } + bytes.putCodePoint(c); + continue; + } + if(iter + 2 >= value.length()) { + throw new IOException(key + " has a malformed percent escape"); + } + int hi = OtlpSchema.hexDigit(value.charAt(iter + 1)); + int lo = OtlpSchema.hexDigit(value.charAt(iter + 2)); + if(hi < 0 || lo < 0) { + throw new IOException(key + " has a malformed percent escape"); + } + bytes.put((hi << 4) | lo); + iter += 2; + } + // Well-formed UTF-8, or refused. Decoding replaced a bad sequence + // ("orders%C3%28") with U+FFFD, and resource attributes meet no later check, + // so the tracer started and exported a mangled service.name. A decode that + // re-encodes to anything but the same bytes was not well formed: a bad + // sequence comes back as the replacement character, an overlong one as + // its shorter spelling. + byte[] raw = new byte[bytes.length()]; + System.arraycopy(bytes.bytes(), 0, raw, 0, raw.length); + String decoded = new String(raw, "UTF-8"); + if(!java.util.Arrays.equals(raw, decoded.getBytes("UTF-8"))) { + throw new IOException(key + " has a percent escape that is not well-formed UTF-8"); + } + return decoded; + } +} diff --git a/vm/backend/src/com/codename1/backend/otel/Sampler.java b/vm/backend/src/com/codename1/backend/otel/Sampler.java new file mode 100644 index 00000000000..4447f242eb4 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/otel/Sampler.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.otel; + +import java.io.IOException; + +/** + * The OpenTelemetry samplers named by {@code OTEL_TRACES_SAMPLER}: + * {@code always_on}, {@code always_off}, {@code traceidratio} and the three + * {@code parentbased_} forms, which follow the caller's decision when there is a + * caller and apply the named rule only to a new trace. + * + *

Parent-based is the default, and it matters most for a mobile backend: the + * app decides once, at the edge, and every service the request touches agrees, + * so a trace is either whole or absent rather than missing its middle. + */ +final class Sampler { + private final boolean parentBased; + /** For a new trace: 1 always, 0 never, otherwise a ratio. */ + private final double ratio; + private final long bound; + + private Sampler(boolean parentBased, double ratio) { + this.parentBased = parentBased; + this.ratio = ratio; + this.bound = ratio >= 1 ? Long.MAX_VALUE : (long)(ratio * (double)Long.MAX_VALUE); + } + + /** From the sampler's name and its argument, which is the ratio for the ratio forms. */ + static Sampler parse(String name, String argument) throws IOException { + String n = name == null || name.trim().length() == 0 ? "parentbased_always_on" : name.trim(); + if("always_on".equals(n)) { + return new Sampler(false, 1); + } + if("always_off".equals(n)) { + return new Sampler(false, 0); + } + if("traceidratio".equals(n)) { + return new Sampler(false, ratio(argument)); + } + if("parentbased_always_on".equals(n)) { + return new Sampler(true, 1); + } + if("parentbased_always_off".equals(n)) { + return new Sampler(true, 0); + } + if("parentbased_traceidratio".equals(n)) { + return new Sampler(true, ratio(argument)); + } + // Refused rather than defaulted: a deployment that asked for a sampler + // this does not know believes it is getting it. + throw new IOException("Unknown sampler '" + n + "'. Use always_on, always_off, " + + "traceidratio, parentbased_always_on, parentbased_always_off or " + + "parentbased_traceidratio."); + } + + /** + * The ratio argument, read ONLY by the samplers that take one. A shared + * deployment template sets OTEL_TRACES_SAMPLER_ARG for whichever sampler it + * expects, and a service that chose always_on must not refuse to start over an + * argument it never uses. Absent means 1, as the specification says. + */ + private static double ratio(String argument) throws IOException { + if(argument == null || argument.trim().length() == 0) { + return 1; + } + double ratio; + try { + ratio = Double.parseDouble(argument.trim()); + } catch (NumberFormatException err) { + throw new IOException("The sampler argument must be a number between 0 and 1 " + + "and is '" + argument + "'"); + } + if(!(ratio >= 0 && ratio <= 1)) { + throw new IOException("The sampler argument must be between 0 and 1 and is " + + argument); + } + return ratio; + } + + /** + * Whether to record a span. + * + *

The ratio compares the LOW 64 bits of the trace id, which is what the + * other OpenTelemetry SDKs compare, so every service in a trace that samples + * at the same ratio makes the same decision about it without being told. + */ + boolean sample(boolean hasParent, boolean parentSampled, long traceLo) { + if(parentBased && hasParent) { + return parentSampled; + } + if(ratio >= 1) { + return true; + } + if(ratio <= 0) { + return false; + } + return (traceLo & Long.MAX_VALUE) < bound; + } +} diff --git a/vm/backend/src/com/codename1/backend/otel/TraceContext.java b/vm/backend/src/com/codename1/backend/otel/TraceContext.java new file mode 100644 index 00000000000..5d79aa3a169 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/otel/TraceContext.java @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.otel; + +/** + * W3C Trace Context (https://www.w3.org/TR/trace-context/): parsing a + * {@code traceparent}, writing one, and vetting a {@code tracestate}. + * + *

Parsing is STRICT the way the specification asks, because a header that + * arrives from the internet is input, and the answer to a malformed one is always + * the same: ignore it and start a new trace. Upper-case hex, an all-zero id, the + * forbidden version ff and a version-00 header with anything after the flags are + * all refused. + */ +final class TraceContext { + final long traceHi; + final long traceLo; + final long spanId; + final int flags; + + TraceContext(long traceHi, long traceLo, long spanId, int flags) { + this.traceHi = traceHi; + this.traceLo = traceLo; + this.spanId = spanId; + this.flags = flags; + } + + boolean sampled() { + return (flags & 1) != 0; + } + + /** The header, or null when it is not a valid one. */ + static TraceContext parse(String header) { + if(header == null) { + return null; + } + String h = header.trim(); + // version "-" trace-id "-" parent-id "-" flags: 2+1+32+1+16+1+2 + if(h.length() < 55) { + return null; + } + int version = hexByte(h, 0); + if(version < 0 || version == 0xff || h.charAt(2) != '-' || h.charAt(35) != '-' + || h.charAt(52) != '-') { + return null; + } + if(version == 0 && h.length() != 55) { + return null; + } + // A LATER version may append fields, and a parser of this version reads + // the ones it knows -- but only if the next character ends the flags. + if(version > 0 && h.length() > 55 && h.charAt(55) != '-') { + return null; + } + long hi = hex64(h, 3); + long lo = hex64(h, 19); + long span = hex64(h, 36); + int flags = hexByte(h, 53); + if(hi == BAD || lo == BAD || span == BAD || flags < 0) { + return null; + } + if((hi == 0 && lo == 0) || span == 0) { + return null; + } + return new TraceContext(hi, lo, span, flags); + } + + /** Sixteen lower-case hex digits at {@code from}, or {@link #BAD}. */ + private static final long BAD = 0x7fffffffffffffffL; + + private static long hex64(String text, int from) { + long value = 0; + for(int iter = 0 ; iter < 16 ; iter++) { + int d = lowerHex(text.charAt(from + iter)); + if(d < 0) { + // BAD is itself a legal id, so a real one that happens to equal it + // would be refused; one in 2^64, and the cost is a new trace. + return BAD; + } + value = (value << 4) | d; + } + return value; + } + + private static int hexByte(String text, int from) { + int hi = lowerHex(text.charAt(from)); + int lo = lowerHex(text.charAt(from + 1)); + return hi < 0 || lo < 0 ? -1 : (hi << 4) | lo; + } + + private static int lowerHex(char c) { + if(c >= '0' && c <= '9') { + return c - '0'; + } + if(c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + return -1; + } + + /** {@code 00---}. */ + static String format(long traceHi, long traceLo, long spanId, boolean sampled) { + StringBuilder out = new StringBuilder(55); + out.append("00-"); + appendHex(out, traceHi); + appendHex(out, traceLo); + out.append('-'); + appendHex(out, spanId); + out.append(sampled ? "-01" : "-00"); + return out.toString(); + } + + private static final char[] DIGITS = "0123456789abcdef".toCharArray(); + + static void appendHex(StringBuilder out, long value) { + for(int shift = 60 ; shift >= 0 ; shift -= 4) { + out.append(DIGITS[(int)((value >>> shift) & 0xf)]); + } + } + + static String hex(long value) { + StringBuilder out = new StringBuilder(16); + appendHex(out, value); + return out.toString(); + } + + /** + * The tracestate to carry on, or null. It is opaque to this server -- vendors + * keep their own state in it -- so a valid one is passed through exactly as it + * came. A header that is not a valid list is DISCARDED whole, which is what the + * specification requires of a receiver: forwarding it would hand the next hop a + * value a strict parser rejects, and with it every vendor's entry. + * + * Valid means at most 32 members, each {@code key=value} with the grammar the + * specification gives (below), no key twice, and no more than 512 characters in + * all. + * + * EMPTY MEMBERS ARE ACCEPTED, NOT REJECTED: the specification says "empty and + * whitespace-only list members are allowed" and that vendors MUST accept them, + * because several tracestate header lines joined by an intermediary produce + * exactly that. It also says a vendor SHOULD avoid sending them, so the list is + * passed on rebuilt without them rather than exactly as it arrived. + */ + static String vetTracestate(String state) { + if(state == null) { + return null; + } + String s = state.trim(); + if(s.length() == 0 || s.length() > 512) { + return null; + } + java.util.HashSet keys = new java.util.HashSet(); + StringBuilder rebuilt = new StringBuilder(s.length()); + int members = 0; + int at = 0; + while(at <= s.length()) { + int comma = s.indexOf(',', at); + if(comma < 0) { + comma = s.length(); + } + String member = s.substring(at, comma).trim(); + at = comma + 1; + if(member.length() == 0) { + continue; + } + int eq = member.indexOf('='); + if(eq <= 0 || !validKey(member.substring(0, eq)) + || !validValue(member.substring(eq + 1))) { + return null; + } + if(!keys.add(member.substring(0, eq)) || ++members > 32) { + return null; + } + if(rebuilt.length() > 0) { + rebuilt.append(','); + } + rebuilt.append(member); + } + return members == 0 ? null : rebuilt.toString(); + } + + /** + * {@code simple-key = lcalpha 0*255( lcalpha / DIGIT / "_" / "-"/ "*" / "/" )}, or + * {@code multi-tenant-key = tenant-id "@" system-id} where the tenant is + * {@code ( lcalpha / DIGIT ) 0*240( ... )} and the system + * {@code lcalpha 0*13( ... )}. + */ + static boolean validKey(String key) { + int at = key.indexOf('@'); + if(at < 0) { + return key.length() <= 256 && isLower(key.charAt(0)) && keyChars(key, 1); + } + String tenant = key.substring(0, at); + String system = key.substring(at + 1); + return tenant.length() >= 1 && tenant.length() <= 241 + && (isLower(tenant.charAt(0)) || isDigit(tenant.charAt(0))) && keyChars(tenant, 1) + && system.length() >= 1 && system.length() <= 14 + && isLower(system.charAt(0)) && keyChars(system, 1); + } + + private static boolean keyChars(String text, int from) { + for(int iter = from ; iter < text.length() ; iter++) { + char c = text.charAt(iter); + if(!isLower(c) && !isDigit(c) && c != '_' && c != '-' && c != '*' && c != '/') { + return false; + } + } + return true; + } + + /** + * {@code value = 0*255(chr) nblk-chr}: printable ASCII except comma and equals, + * spaces allowed inside but not at the end, 256 characters at most. + */ + static boolean validValue(String value) { + if(value.length() == 0 || value.length() > 256) { + return false; + } + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + if(c < 0x20 || c > 0x7e || c == ',' || c == '=') { + return false; + } + } + return value.charAt(value.length() - 1) != ' '; + } + + private static boolean isLower(char c) { + return c >= 'a' && c <= 'z'; + } + + private static boolean isDigit(char c) { + return c >= '0' && c <= '9'; + } +} diff --git a/vm/tests/pom.xml b/vm/tests/pom.xml index 1f04ff1e280..d822f440aaf 100644 --- a/vm/tests/pom.xml +++ b/vm/tests/pom.xml @@ -67,6 +67,22 @@ 1.6.0 test + + + io.opentelemetry.proto + opentelemetry-proto + 1.3.2-alpha + test + + + com.google.protobuf + protobuf-java + 3.25.5 + test + diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BackendOtelTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BackendOtelTest.java new file mode 100644 index 00000000000..ce7cc2a650b --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BackendOtelTest.java @@ -0,0 +1,483 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import com.google.protobuf.ByteString; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; +import io.opentelemetry.proto.trace.v1.Span; +import io.opentelemetry.proto.trace.v1.Status; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * OpenTelemetry tracing on the PACKAGED runtime. + * + *

The JVM arm has its own end-to-end test in the backend module; this one is + * for what only a translated binary can get wrong. The encoder, the id + * generator and the exporter thread all run as translated C here, outbound HTTP is + * libcurl -- whose redirect rule had to learn that trace headers are not the + * caller's -- and the claim that an untraced server carries no tracer is a claim + * about what the translator leaves in the binary, which only a binary can answer. + */ +class BackendOtelTest { + private static final String TRACE = "4bf92f3577b34da6a3ce929d0e0e4736"; + private static final String CALLER_SPAN = "00f067aa0ba902b7"; + /** The trace the HTTP/2 request carries, so its span is told from the HTTP/1 ones. */ + private static final String H2_TRACE = "0af7651916cd43dd8448eb211c80319c"; + /** The traces the websocket handshakes carry: one accepted, one refused. */ + private static final String WS_TRACE = "5b8aa5a2d2c872e8321cf37308d69df2"; + private static final String WS_REFUSED_TRACE = "7d0a1e4bb7c9a2f35e61d8c04f2b9a13"; + + @Test + @DisplayName("a translated server exports one connected trace, and an untraced one carries no tracer") + void tracesOnThePackagedRuntime() throws Exception { + if (CompilerHelper.isWindows()) { + BackendTestSupport.skipOrFail("the server-side backend is POSIX-only for now"); + } + Path jdk8 = BackendTestSupport.findJdk8(); + BackendTestSupport.require(jdk8 != null, "no JDK 8 available to compile the backend"); + BackendTestSupport.require(BackendTestSupport.hasCommand("nm"), + "nm is needed to read the binaries' symbols"); + Path work = Files.createTempDirectory("backend-otel"); + Path traced = work.resolve("otelserver"); + String failure = BackendTestSupport.build("OtelServer", "demo/oteltest", traced, jdk8); + if (failure != null) { + BackendTestSupport.skipOrFail(failure); + } + + // THE ZERO-COST CLAIM, from the binaries. WebCheck is a program that never + // mentions a tracer; it is compiled against the same runtime tree -- the + // otel package included -- so the only thing keeping the tracer out of it + // is the translator's reachability. The traced binary is the control: a + // symbol spelling that matched nothing would pass the first assertion for + // any build. + Path untraced = work.resolve("webcheck"); + failure = BackendTestSupport.build("WebCheck", "demo/webcheck", untraced, jdk8); + if (failure != null) { + BackendTestSupport.skipOrFail(failure); + } + assertTrue(otelSymbols(traced) > 0, + "the traced binary carries no tracer symbols; the check below would be vacuous"); + assertEquals(0, otelSymbols(untraced), + "a server that never asked for tracing links the tracer anyway"); + + final List exports = Collections.synchronizedList(new ArrayList()); + final List contentTypes = Collections.synchronizedList(new ArrayList()); + final List authorizations = Collections.synchronizedList(new ArrayList()); + final Map downstreamHeaders = Collections.synchronizedMap(new HashMap()); + HttpServer peer = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + peer.createContext("/v1/traces", (HttpExchange exchange) -> { + exports.add(readAll(exchange.getRequestBody())); + contentTypes.add(exchange.getRequestHeaders().getFirst("Content-Type")); + authorizations.add(String.valueOf(exchange.getRequestHeaders().getFirst("Authorization"))); + exchange.sendResponseHeaders(200, -1); + exchange.close(); + }); + peer.createContext("/hop", (HttpExchange exchange) -> { + exchange.getResponseHeaders().add("Location", "/downstream"); + exchange.sendResponseHeaders(302, -1); + exchange.close(); + }); + peer.createContext("/downstream", (HttpExchange exchange) -> { + downstreamHeaders.put("traceparent", exchange.getRequestHeaders().getFirst("traceparent")); + downstreamHeaders.put("tracestate", exchange.getRequestHeaders().getFirst("tracestate")); + byte[] body = "down".getBytes("UTF-8"); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + peer.start(); + String peerBase = "http://127.0.0.1:" + peer.getAddress().getPort(); + + int port = BackendTestSupport.freePort(); + Map env = new HashMap(); + env.put("PORT", String.valueOf(port)); + env.put("OTEL_EXPORTER_OTLP_ENDPOINT", peerBase); + env.put("OTEL_EXPORTER_OTLP_HEADERS", "Authorization=Api-Token%20t0k"); + env.put("OTEL_BSP_SCHEDULE_DELAY", "200"); + env.put("CN1_OTEL_RELAY", "true"); + env.put("CN1_OTEL_DOWNSTREAM", peerBase); + Path log = work.resolve("server.log"); + Process server = BackendTestSupport.start(traced, env, log); + try { + assertTrue(BackendTestSupport.waitForPort(port, 30000), + "the traced server never listened:\n" + read(log)); + HttpURLConnection request = (HttpURLConnection) new URL( + "http://127.0.0.1:" + port + "/work?token=secret").openConnection(); + request.setRequestProperty("traceparent", "00-" + TRACE + "-" + CALLER_SPAN + "-01"); + request.setRequestProperty("tracestate", "vendor=opaque"); + assertEquals(200, request.getResponseCode(), read(log)); + // "Rex" from the database, and the outbound call FOLLOWED the redirect + // to reach "down": carrying trace headers did not make libcurl treat + // the call as one with credentials to protect. + assertEquals("Rex 200 down", new String(readAll(request.getInputStream()), "UTF-8")); + + HttpURLConnection boom = (HttpURLConnection) new URL( + "http://127.0.0.1:" + port + "/boom").openConnection(); + assertEquals(500, boom.getResponseCode()); + + assertEquals(200, relay(port, "{\"resourceSpans\":[{\"resource\":{\"attributes\":" + + "[{\"key\":\"service.name\",\"value\":{\"stringValue\":\"app\"}}]}," + + "\"scopeSpans\":[{\"spans\":[{\"traceId\":\"" + TRACE + "\",\"spanId\":\"" + + CALLER_SPAN + "\",\"name\":\"tap\",\"kind\":3,\"startTimeUnixNano\":" + + "\"1700000000000000000\",\"endTimeUnixNano\":\"1700000000100000000\"}]}]}]}")); + + // Over HTTP/2 as well: its span ends only after the response is + // written, and :authority -- HTTP/2's Host -- has to reach + // server.address just as Host does. + assertTrue(http2Get(port, "/h2probe", "00-" + H2_TRACE + "-" + CALLER_SPAN + "-01"), + "no HTTP/2 response came back:\n" + read(log)); + + // A websocket handshake is a request with a span of its own, and what + // onOpen does is its child; a refused one is a span with the refusal. + assertEquals("101", handshake(port, "/ws", WS_TRACE), read(log)); + assertEquals("404", handshake(port, "/nows", WS_REFUSED_TRACE), read(log)); + + List spans = awaitSpans(exports, 9, 30000); + assertEquals("application/x-protobuf", contentTypes.get(0)); + assertEquals("Api-Token t0k", authorizations.get(0)); + + Span work0 = find(spans, "GET /work", Span.SpanKind.SPAN_KIND_SERVER); + Span query = null; + for (int iter = 0; iter < spans.size(); iter++) { + Span s = (Span) spans.get(iter); + if ("SELECT".equals(s.getName()) && hex(s.getParentSpanId()).equals(hex(work0.getSpanId()))) { + query = s; + } + } + assertTrue(query != null, "no statement under GET /work in " + spans); + Span outbound = find(spans, "GET", Span.SpanKind.SPAN_KIND_CLIENT); + Span failed = null; + Span overH2 = null; + for (int iter = 0; iter < spans.size(); iter++) { + Span s = (Span) spans.get(iter); + if (s.getKind() != Span.SpanKind.SPAN_KIND_SERVER) { + continue; + } + if (H2_TRACE.equals(hex(s.getTraceId()))) { + overH2 = s; + } else if (s.getStatus().getCode() == Status.StatusCode.STATUS_CODE_ERROR) { + failed = s; + } + } + Span accepted = null; + Span refused = null; + Span onOpenQuery = null; + for (int iter = 0; iter < spans.size(); iter++) { + Span s = (Span) spans.get(iter); + String trace = hex(s.getTraceId()); + if (WS_TRACE.equals(trace) && s.getKind() == Span.SpanKind.SPAN_KIND_SERVER) { + accepted = s; + } else if (WS_TRACE.equals(trace)) { + onOpenQuery = s; + } else if (WS_REFUSED_TRACE.equals(trace)) { + refused = s; + } + } + assertTrue(accepted != null, "the accepted handshake has no span: " + spans); + assertEquals("101", attribute(accepted.getAttributesList(), "http.response.status_code")); + assertEquals("/ws", attribute(accepted.getAttributesList(), "url.path")); + // Named after the endpoint it reached, as an HTTP route is: otherwise + // every endpoint's handshake is one operation called "GET". + assertEquals("GET /ws", accepted.getName()); + assertEquals("/ws", attribute(accepted.getAttributesList(), "http.route")); + assertTrue(onOpenQuery != null, "onOpen's statement is not in the handshake's trace"); + assertEquals(hex(accepted.getSpanId()), hex(onOpenQuery.getParentSpanId()), + "onOpen's work is a child of the handshake"); + assertTrue(refused != null, "the refused handshake has no span: " + spans); + assertEquals("404", attribute(refused.getAttributesList(), "http.response.status_code")); + assertTrue(failed != null, "no failed server span in " + spans); + assertTrue(overH2 != null, "no span for the HTTP/2 request in " + spans); + assertEquals(CALLER_SPAN, hex(overH2.getParentSpanId())); + assertEquals("2", attribute(overH2.getAttributesList(), "network.protocol.version")); + assertEquals("127.0.0.1", attribute(overH2.getAttributesList(), "server.address"), + "HTTP/2's :authority did not reach server.address"); + assertEquals("404", attribute(overH2.getAttributesList(), "http.response.status_code"), + "the span of a written HTTP/2 response carries the status sent"); + Span relayed = find(spans, "tap", Span.SpanKind.SPAN_KIND_CLIENT); + + assertEquals(TRACE, hex(work0.getTraceId())); + assertEquals(CALLER_SPAN, hex(work0.getParentSpanId())); + assertEquals("vendor=opaque", work0.getTraceState()); + assertEquals("/work", attribute(work0.getAttributesList(), "url.path")); + assertEquals("/work", attribute(work0.getAttributesList(), "http.route")); + assertEquals(hex(work0.getSpanId()), hex(query.getParentSpanId())); + assertEquals("sqlite", attribute(query.getAttributesList(), "db.system")); + assertEquals(hex(work0.getSpanId()), hex(outbound.getParentSpanId())); + assertTrue(work0.getEndTimeUnixNano() >= outbound.getEndTimeUnixNano()); + + // What actually went over the wire to the service the call reached. + assertEquals("00-" + TRACE + "-" + hex(outbound.getSpanId()) + "-01", + downstreamHeaders.get("traceparent")); + assertEquals("vendor=opaque", downstreamHeaders.get("tracestate")); + + assertEquals(Status.StatusCode.STATUS_CODE_ERROR, failed.getStatus().getCode()); + assertEquals("java.lang.IllegalStateException", + attribute(failed.getEvents(0).getAttributesList(), "exception.type")); + + assertEquals(TRACE, hex(relayed.getTraceId()), + "the app's span, relayed through the translated server"); + } finally { + BackendTestSupport.stop(server); + peer.stop(0); + } + } + + /** How many translated symbols of the tracer package a binary holds. */ + private static int otelSymbols(Path binary) throws Exception { + String symbols = BackendTestSupport.run(Arrays.asList("nm", binary.toString()), 120); + int count = 0; + int at = 0; + while ((at = symbols.indexOf("com_codename1_backend_otel_", at)) >= 0) { + count++; + at++; + } + return count; + } + + private static List awaitSpans(List exports, int wanted, long timeoutMillis) throws Exception { + long deadline = System.currentTimeMillis() + timeoutMillis; + List spans = new ArrayList(); + while (System.currentTimeMillis() < deadline) { + spans = new ArrayList(); + Object[] snapshot = exports.toArray(); + for (int iter = 0; iter < snapshot.length; iter++) { + ExportTraceServiceRequest request = + ExportTraceServiceRequest.parseFrom((byte[]) snapshot[iter]); + for (ResourceSpans rs : request.getResourceSpansList()) { + for (ScopeSpans ss : rs.getScopeSpansList()) { + spans.addAll(ss.getSpansList()); + } + } + } + if (spans.size() >= wanted) { + return spans; + } + Thread.sleep(100); + } + throw new AssertionError("expected " + wanted + " spans, the collector received " + + spans.size() + ": " + spans); + } + + /** + * A websocket handshake carrying a traceparent; the status code the server + * answered, as text. The connection is closed as soon as the status line is in. + */ + private static String handshake(int port, String path, String trace) throws IOException { + java.net.Socket socket = new java.net.Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", port), 5000); + socket.setSoTimeout(10000); + try { + OutputStream out = socket.getOutputStream(); + out.write(("GET " + path + " HTTP/1.1\r\n" + + "Host: 127.0.0.1\r\n" + + "Upgrade: websocket\r\n" + + "Connection: Upgrade\r\n" + + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + + "Sec-WebSocket-Version: 13\r\n" + + "traceparent: 00-" + trace + "-" + CALLER_SPAN + "-01\r\n" + + "\r\n").getBytes("UTF-8")); + out.flush(); + InputStream in = socket.getInputStream(); + StringBuilder line = new StringBuilder(); + int c; + while ((c = in.read()) >= 0 && c != '\n') { + line.append((char) c); + } + String status = line.toString().trim(); + int space = status.indexOf(' '); + return space < 0 ? status : status.substring(space + 1, Math.min(status.length(), space + 4)); + } finally { + socket.close(); + } + } + + /** + * One GET over cleartext HTTP/2 by prior knowledge, carrying a traceparent; + * whether a response HEADERS frame came back. The frame layout is + * BackendHttpIntegrationTest's. + */ + private static boolean http2Get(int port, String path, String traceparent) throws IOException { + java.net.Socket socket = new java.net.Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", port), 5000); + socket.setSoTimeout(10000); + try { + OutputStream out = socket.getOutputStream(); + out.write("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n".getBytes("UTF-8")); + out.write(h2Frame(4, 0, 0, new byte[0])); + ByteArrayOutputStream block = new ByteArrayOutputStream(); + hpackLiteral(block, ":method", "GET"); + hpackLiteral(block, ":path", path); + hpackLiteral(block, ":scheme", "http"); + hpackLiteral(block, ":authority", "127.0.0.1"); + hpackLiteral(block, "traceparent", traceparent); + out.write(h2Frame(1, 0x05, 1, block.toByteArray())); + out.flush(); + InputStream in = socket.getInputStream(); + long deadline = System.currentTimeMillis() + 8000; + while (System.currentTimeMillis() < deadline) { + byte[] header = readExactly(in, 9); + if (header == null) { + return false; + } + int length = ((header[0] & 0xff) << 16) | ((header[1] & 0xff) << 8) | (header[2] & 0xff); + if (length > 0 && readExactly(in, length) == null) { + return false; + } + if ((header[3] & 0xff) == 1) { + return true; + } + } + return false; + } finally { + socket.close(); + } + } + + private static byte[] h2Frame(int type, int flags, int streamId, byte[] payload) { + byte[] out = new byte[9 + payload.length]; + out[0] = (byte) ((payload.length >>> 16) & 0xff); + out[1] = (byte) ((payload.length >>> 8) & 0xff); + out[2] = (byte) (payload.length & 0xff); + out[3] = (byte) type; + out[4] = (byte) flags; + out[5] = (byte) ((streamId >>> 24) & 0x7f); + out[6] = (byte) ((streamId >>> 16) & 0xff); + out[7] = (byte) ((streamId >>> 8) & 0xff); + out[8] = (byte) (streamId & 0xff); + System.arraycopy(payload, 0, out, 9, payload.length); + return out; + } + + /** An uncompressed HPACK literal, the one form every decoder accepts. */ + private static void hpackLiteral(ByteArrayOutputStream out, String name, String value) + throws IOException { + byte[] n = name.getBytes("ISO-8859-1"); + byte[] v = value.getBytes("ISO-8859-1"); + out.write(0x00); + out.write(n.length); + out.write(n); + out.write(v.length); + out.write(v); + } + + private static byte[] readExactly(InputStream in, int count) throws IOException { + byte[] out = new byte[count]; + int filled = 0; + while (filled < count) { + int n = in.read(out, filled, count - filled); + if (n < 0) { + return null; + } + filled += n; + } + return out; + } + + private static Span find(List spans, String name, Span.SpanKind kind) { + for (int iter = 0; iter < spans.size(); iter++) { + Span span = (Span) spans.get(iter); + if (name.equals(span.getName()) && span.getKind() == kind) { + return span; + } + } + throw new AssertionError("no " + kind + " span named " + name + " in " + spans); + } + + private static String attribute(List attributes, String key) { + for (int iter = 0; iter < attributes.size(); iter++) { + KeyValue kv = (KeyValue) attributes.get(iter); + if (kv.getKey().equals(key)) { + return kv.getValue().hasIntValue() + ? String.valueOf(kv.getValue().getIntValue()) + : kv.getValue().getStringValue(); + } + } + return null; + } + + private static String hex(ByteString bytes) { + StringBuilder out = new StringBuilder(); + for (int iter = 0; iter < bytes.size(); iter++) { + out.append(String.format("%02x", bytes.byteAt(iter) & 0xff)); + } + return out.toString(); + } + + private static int relay(int port, String json) throws IOException { + HttpURLConnection connection = (HttpURLConnection) new URL( + "http://127.0.0.1:" + port + "/otel/v1/traces").openConnection(); + connection.setRequestMethod("POST"); + connection.setDoOutput(true); + connection.setRequestProperty("Content-Type", "application/json"); + OutputStream out = connection.getOutputStream(); + out.write(json.getBytes("UTF-8")); + out.close(); + return connection.getResponseCode(); + } + + private static String read(Path log) { + try { + return new String(Files.readAllBytes(log), "UTF-8"); + } catch (IOException err) { + return "(no log: " + err + ")"; + } + } + + private static byte[] readAll(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] chunk = new byte[4096]; + int n; + while ((n = in.read(chunk)) > 0) { + out.write(chunk, 0, n); + } + in.close(); + return out.toByteArray(); + } +}