diff --git a/.gitignore b/.gitignore index 92e132338..61652c446 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .gradle -/local.properties +local.properties .DS_Store build /captures diff --git a/example/build.gradle b/example/build.gradle index f5286b24b..b29f6a623 100644 --- a/example/build.gradle +++ b/example/build.gradle @@ -2,8 +2,6 @@ import java.util.Properties plugins { id("com.android.application") - // make sure this line comes *after* you apply the Android plugin - id("com.getkeepsafe.dexcount") } // local.properties is not checked in, so it is where machine-specific settings such as your mobile diff --git a/example/src/main/java/com/launchdarkly/example/MainActivity.java b/example/src/main/java/com/launchdarkly/example/MainActivity.java index 9b6043240..512be492c 100644 --- a/example/src/main/java/com/launchdarkly/example/MainActivity.java +++ b/example/src/main/java/com/launchdarkly/example/MainActivity.java @@ -26,10 +26,6 @@ import java.util.Date; import java.util.Locale; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import timber.log.Timber; @@ -45,6 +41,9 @@ public class MainActivity extends AppCompatActivity { private static final String DEFAULT_USER_KEY = "user key"; + /** How long startup blocks waiting for the first flags to arrive. */ + private static final int INIT_WAIT_SECONDS = 10; + private LDClient ldClient; private LDStatusListener ldStatusListener; private LDAllFlagsListener allFlagsListener; @@ -142,15 +141,12 @@ public void onCreate(Bundle savedInstanceState) { .set("email", "fake@example.com") .build(); - Future initFuture = LDClient.init(this.getApplication(), ldConfig, context); - try { - ldClient = initFuture.get(10, TimeUnit.SECONDS); - updateStatusString(ldClient.getConnectionInformation()); - ldClient.registerStatusListener(ldStatusListener); - ldClient.registerAllFlagsListener(allFlagsListener); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - Timber.e(e, "Exception when awaiting LaunchDarkly Client initialization"); - } + // Returns the client either way: if the flags have not arrived within the wait, it is usable + // with whatever it has cached. + ldClient = LDClient.init(this.getApplication(), ldConfig, context, INIT_WAIT_SECONDS); + updateStatusString(ldClient.getConnectionInformation()); + ldClient.registerStatusListener(ldStatusListener); + ldClient.registerAllFlagsListener(allFlagsListener); } private void setupListeners() { diff --git a/launchdarkly-android-client-sdk/build.gradle b/launchdarkly-android-client-sdk/build.gradle index 3b7566cb1..f59fce991 100644 --- a/launchdarkly-android-client-sdk/build.gradle +++ b/launchdarkly-android-client-sdk/build.gradle @@ -27,10 +27,30 @@ android { // that the app's state is completely cleared between tests. testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" testInstrumentationRunnerArguments["clearPackageData"] = "true" + if (System.getenv("LD_EVENT_BENCH") == "1") { + // The device benchmark writes one results file for the class. Orchestrator's + // "pm clear" between methods would wipe it, and the runner does not inherit the + // shell environment, so the gate is passed as an instrumentation argument. + testInstrumentationRunnerArguments["clearPackageData"] = "false" + testInstrumentationRunnerArguments["LD_EVENT_BENCH"] = "1" + } } testOptions { - execution = "ANDROIDX_TEST_ORCHESTRATOR" + // EventPersistenceBenchmark on a device keeps one process for the class so its results + // file survives; orchestrator would "pm clear" between methods and wipe it. + execution = (System.getenv("LD_EVENT_BENCH") == "1") ? "HOST" : "ANDROIDX_TEST_ORCHESTRATOR" + + // EventPersistenceBenchmark reports its measurements on stdout, which Gradle otherwise + // swallows, and it is worth re-running even when nothing has changed. Both only apply when + // LD_EVENT_BENCH is set, which is also what un-skips the benchmark, so an ordinary test run + // is unaffected. + unitTests.all { test -> + if (System.getenv("LD_EVENT_BENCH") == "1") { + test.testLogging.showStandardStreams = true + test.outputs.upToDateWhen { false } + } + } } buildFeatures { @@ -79,7 +99,7 @@ ext.versions = [ "jacksonDatabind": "2.10.5.1", "junit": "4.13", "launchdarklyJavaSdkCommon": "2.4.0", - "launchdarklyJavaSdkInternal": "1.9.0", + "launchdarklyJavaSdkInternal": "1.12.1", "launchdarklyLogging": "1.1.1", "okhttp": "4.12.0", "testHelpers": "2.1.0", diff --git a/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/EventPersistenceBenchmark.java b/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/EventPersistenceBenchmark.java new file mode 100644 index 000000000..c27d92934 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/EventPersistenceBenchmark.java @@ -0,0 +1,659 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assume.assumeTrue; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.AttributeRef; +import com.launchdarkly.sdk.ContextKind; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.internal.events.Event; + +import androidx.test.platform.app.InstrumentationRegistry; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.IntConsumer; + +/** + * The Android counterpart of the iOS {@code EventPersistenceBenchmark}: what it costs to record an + * analytics event, so the encoding research can be argued from measurements on this platform rather + * than from the ones taken on iOS. + *

+ * This exists because the two platforms are not obviously comparable. iOS encodes with {@code Codable}, + * Android encodes with {@code EventOutputFormatter} in java-sdk-internal, and a JIT with escape + * analysis behaves nothing like ahead-of-time compiled Swift. The iOS finding -- that the cost is in + * the serializer's per-value machinery rather than in walking and redacting the context -- is a claim + * about {@code Codable}, and nothing about it transfers here by assumption. + *

+ * Instrumented copy of the JVM {@code EventPersistenceBenchmark}: same corpus, same loops, running + * on ART on a device rather than on a desktop JVM. Skipped unless the instrumentation argument + * {@code LD_EVENT_BENCH=1} is set, because nothing here asserts anything a regression would trip: + * 

+ *   ./gradlew :launchdarkly-android-client-sdk:connectedDebugAndroidTest \\
+ *       -Pandroid.testInstrumentationRunnerArguments.LD_EVENT_BENCH=1 \\
+ *       -Pandroid.testInstrumentationRunnerArguments.class=com.launchdarkly.sdk.android.EventPersistenceBenchmark
+ *
+ * Results are written to {@code event-bench.txt} in the target app's files directory (pulled after + * the run) as well as to logcat, because Gradle's instrumented-test logger does not reliably keep + * {@code System.out}. + */ +public class EventPersistenceBenchmark { + private static final String FLAG_KEY = "benchmark-flag"; + private static final int FLAG_VERSION = 7; + private static final int VARIATION = 1; + + /** + * Fixed rather than {@code System.currentTimeMillis()}, so that repeated runs encode identical + * bytes and the clock call is not charged to the encoder. The event-construction row below covers + * what the real path pays around it. + */ + private static final long CREATION_DATE = 1_700_000_000_000L; + + /** + * One already serialized event, byte for byte what the iOS benchmark stages, so the store figures on the + * two platforms are measuring the same amount of work reaching the same kind of file. + */ + private static final String SERIALIZED_EVENT = + "{\"kind\":\"feature\",\"key\":\"benchmark-flag\",\"value\":true,\"default\":false," + + "\"variation\":1,\"version\":7,\"creationDate\":1740000000000}"; + + /** How many times each timed loop is repeated; the fastest round is reported. */ + private static final int ROUNDS = 5; + + /** + * The sizes the commit-point figures are swept over. The same three are used by the iOS benchmark, so + * the two platforms' tables line up row for row. + */ + private static final int[] BULK_SIZES = {100, 1_000, 10_000}; + + /** + * Consumes results so that the JIT cannot prove the work is unused and delete it. Never read for + * its value. + */ + private static long blackhole; + + /** + * Holds an allocation so that escape analysis cannot scalar-replace it. Assigning to a static field + * is what makes the object genuinely exist, which is the difference between measuring an allocation + * and measuring nothing at all. + */ + private static Object escaped; + + @Rule + public TemporaryFolder eventsDirectory = new TemporaryFolder( + InstrumentationRegistry.getInstrumentation().getTargetContext().getCacheDir()); + + private final LDLogger logger = LDLogger.withAdapter(Logs.none(), ""); + + private static PrintWriter logFile; + + @BeforeClass + public static void openLog() throws IOException { + File file = new File( + InstrumentationRegistry.getInstrumentation().getTargetContext().getFilesDir(), + "event-bench.txt"); + logFile = new PrintWriter(new FileWriter(file, true), true); + logFile.println("device=" + android.os.Build.MODEL + + " manufacturer=" + android.os.Build.MANUFACTURER + + " device=" + android.os.Build.DEVICE + + " abi=" + android.os.Build.SUPPORTED_ABIS[0] + + " sdk=" + android.os.Build.VERSION.SDK_INT + + " release=" + android.os.Build.VERSION.RELEASE + + " hardware=" + android.os.Build.HARDWARE); + } + + @AfterClass + public static void closeLog() { + if (logFile != null) { + logFile.close(); + } + } + + @Before + public void requireBenchmarking() { + assumeTrue("Set instrumentation argument LD_EVENT_BENCH=1 to measure event recording", + "1".equals(InstrumentationRegistry.getArguments().getString("LD_EVENT_BENCH"))); + } + + /** + * M1 and M2 for Android: where the time goes when one feature event is serialized, across the + * context shapes and privacy settings that change how much work redaction has to do. + *

+ * The shape of the result is the interesting part. If the cost is dominated by walking and + * redacting the context, it should grow with the number of attributes and grow again when private + * attributes force the filtering path. If it is dominated by the serializer's fixed per-event + * machinery, it should be far flatter than the attribute counts suggest. + */ + @Test + public void encodingCostByContextShape() { + List building = new ArrayList<>(); + List serializing = new ArrayList<>(); + + for (ContextShape shape : ContextShape.values()) { + final LDContext context = makeContext(shape, "benchmark-key"); + + building.add(new Measurement(shape.label, measure(200_000, i -> { + escaped = featureEvent(context, true); + }))); + + for (PrivacyShape privacy : privacyShapes()) { + final OutboundEventBuffer buffer = makeBuffer(privacy); + + byte[] sample = buffer.serialize(featureEvent(context, true)); + assertNotNull("the corpus produced an event that will not serialize", sample); + + double nanos = measure(50_000, i -> { + byte[] out = buffer.serialize(featureEvent(context, true)); + blackhole += out == null ? 0 : out.length; + }); + serializing.add(new Measurement( + shape.label + ", " + privacy.name + ", " + sample.length + " bytes", nanos)); + } + } + + report("building one feature event, no serialization", building); + report("serializing one feature event", serializing); + } + + /** + * The figures a customer actually pays, mirroring what {@link DirectEventProcessor} does per call. + *

+ * The three rows are the three paths through {@code recordEvaluationEvent} and {@code recordCustomEvent}: + * an evaluation of an untracked flag is a counter increment; an evaluation of a tracked flag also + * serializes and stages a full event; and a {@code track} is a commit point, which additionally turns + * the counters into summary events and writes everything to the log. + *

+ * The sequence is reproduced here rather than driven through the processor itself, which would need + * an HTTP server, a scheduler and a {@code ClientContext} whose costs would swamp what is being + * measured. It is kept deliberately close to the original; if the processor changes, this should too. + */ + @Test + public void recordingCostPerCall() throws IOException { + final OutboundEventBuffer buffer = makeBuffer(noRedaction()); + final EventStore store = EventStore.create(eventsDirectory.getRoot(), "benchmark-key", + "benchmark", Integer.MAX_VALUE, true, logger); + final LDContext context = makeContext(ContextShape.STUB, "benchmark-key"); + + try { + List results = new ArrayList<>(); + + results.add(new Measurement("evaluation, summary only", measure(200_000, i -> { + buffer.summarize(featureEvent(context, false)); + }))); + + results.add(new Measurement("evaluation, full event", measure(50_000, i -> { + Event.FeatureRequest event = featureEvent(context, true); + buffer.summarize(event); + byte[] out = buffer.serialize(event); + if (out != null) { + store.stage(out); + blackhole += out.length; + } + }))); + + results.add(new Measurement("evaluation + track (commit point)", measure(20_000, i -> { + buffer.summarize(featureEvent(context, false)); + byte[] out = buffer.serialize( + new Event.Custom(CREATION_DATE, "benchmark-event", context, LDValue.ofNull(), null)); + if (out != null) { + store.stage(out); + blackhole += out.length; + } + for (byte[] summary : buffer.serializeSummariesAndReset()) { + store.stage(summary, true); + blackhole += summary.length; + } + store.commit(); + }))); + + report("recording, per call", results); + } finally { + store.close(); + } + } + + /** + * What a commit point costs when it goes badly, rather than on average. + *

+ * {@code track} and {@code identify} commit on the caller's thread by design -- that is the durable + * barrier -- so this is the one figure here an application can observe as a stall. The mean is the number + * that gets quoted; the tail is the number a user notices. + *

+ * The counterpart iOS test runs this against a SQLite store as well, where the tail is the point: a + * database in WAL mode folds its write-ahead log back into the file every thousand pages, on whichever + * thread happens to be committing, so one unlucky caller pays for all of them. An append has no such + * thing, and these rows are what that claim is measured against. + *

+ * Two rows per size. {@code track} commits where it stands, so its tail is the write. The other row + * encodes and stages without committing, and the store never writes on its own, so its tail is the + * encode and the copy; the gap between the two is what the write itself costs. + *

+ * p99.9 only separates from the worst at ten thousand samples. Below that the index falls on the + * last sample and the two columns are the same number by construction. They are left side by side so the + * table lines up across the sweep, and so the size at which they part is visible rather than asserted. + */ + @Test + public void commitPointOutliers() throws IOException { + final byte[] serialized = SERIALIZED_EVENT.getBytes(StandardCharsets.UTF_8); + final OutboundEventBuffer buffer = makeBuffer(noRedaction()); + final LDContext context = makeContext(ContextShape.STUB, "benchmark-key"); + + // A store of its own for each size and each row, so that a log inherited from an earlier measurement + // does not charge the next one for the file it had already grown. + for (int calls : BULK_SIZES) { + final EventStore committing = EventStore.create(eventsDirectory.newFolder(), "benchmark-key", + "benchmark", Integer.MAX_VALUE, true, logger); + final EventStore staging = EventStore.create(eventsDirectory.newFolder(), "benchmark-key", + "benchmark", Integer.MAX_VALUE, true, logger); + try { + List results = new ArrayList<>(); + + results.add(new DistributionMeasurement("track, commits on this thread", + measureDistribution(calls, i -> { + committing.stage(serialized); + committing.commit(); + }))); + + results.add(new DistributionMeasurement("evaluation, staged only", + measureDistribution(calls, i -> { + Event.FeatureRequest event = featureEvent(context, true); + buffer.summarize(event); + byte[] out = buffer.serialize(event); + if (out != null) { + staging.stage(out); + blackhole += out.length; + } + }))); + + reportDistribution("one call, " + count(calls) + " of them, on the caller's thread", results); + } finally { + committing.close(); + staging.close(); + } + } + } + + /** + * Whether O2 -- caching the encoded context and reusing it while the context has not changed -- + * could work here, which comes down to what the cache key would cost to check. + *

+ * Android differs from iOS in a way that matters. {@code LDContext} is an immutable object passed by + * reference, and nothing on the event path copies it, so an application that holds one context hands + * the same instance to every event and {@code equals} can settle on reference identity. iOS had to + * reach that fast path by not rebuilding a value type. The row that decides the question is the other + * one: an application that rebuilds its context between evaluations produces an equal object sharing + * nothing, and that comparison is the price of a cache hit in the worst case. + */ + @Test + public void contextComparisonCost() { + List comparisons = new ArrayList<>(); + List hashing = new ArrayList<>(); + + for (ContextShape shape : ContextShape.values()) { + final LDContext context = makeContext(shape, "benchmark-key"); + final LDContext sameInstance = context; + final LDContext rebuilt = makeContext(shape, "benchmark-key"); + + // Both must be equal, or the figures below measure an early exit rather than a comparison. + assertEquals(shape.label, context, sameInstance); + assertEquals(shape.label, context, rebuilt); + + comparisons.add(new Measurement("same instance, " + shape.label, measure(500_000, i -> { + blackhole += context.equals(sameInstance) ? 1 : 0; + }))); + comparisons.add(new Measurement("independently built, " + shape.label, measure(200_000, i -> { + blackhole += context.equals(rebuilt) ? 1 : 0; + }))); + hashing.add(new Measurement(shape.label, measure(500_000, i -> { + blackhole += context.hashCode(); + }))); + } + + report("comparing two equal contexts", comparisons); + report("hashing a context", hashing); + } + + /** + * What the deferred half of the bill costs: summaries are not serialized per evaluation but at + * commit points and deliveries, and the size of that payment depends on how many distinct flags the + * counters have accumulated. + *

+ * Counting and serializing are reported separately, because only the second is deferred -- the + * counter increment is already paid on every evaluation. + */ + @Test + public void summarySerializationCost() { + final LDContext context = makeContext(ContextShape.STUB, "benchmark-key"); + List results = new ArrayList<>(); + + for (final int flagCount : new int[] {1, 10, 50}) { + final OutboundEventBuffer buffer = makeBuffer(noRedaction()); + + double countingOnly = measure(20_000, i -> { + for (int flag = 0; flag < flagCount; flag++) { + buffer.summarize(summaryEvent(context, flag)); + } + }); + // Drained, so the counters left behind above do not inflate the first serialized round. + buffer.serializeSummariesAndReset(); + + double countingAndSerializing = measure(20_000, i -> { + for (int flag = 0; flag < flagCount; flag++) { + buffer.summarize(summaryEvent(context, flag)); + } + for (byte[] summary : buffer.serializeSummariesAndReset()) { + blackhole += summary.length; + } + }); + + results.add(new Measurement(flagCount + " flags: counting only", countingOnly)); + results.add(new Measurement(flagCount + " flags: counting + serializing", + countingAndSerializing)); + results.add(new Measurement(flagCount + " flags: serializing alone", + countingAndSerializing - countingOnly)); + } + + report("turning counters into a summary event", results); + } + + // MARK: Corpus + + /** + * The same five shapes the iOS benchmark uses, so the two sets of numbers describe the same + * contexts. + */ + private enum ContextShape { + KEY_ONLY("key only"), + STUB("stub, 9 attributes"), + WIDE("20 flat attributes"), + NESTED("nested attributes"), + MULTI("multi-context"); + + final String label; + + ContextShape(String label) { + this.label = label; + } + } + + private static LDContext makeContext(ContextShape shape, String key) { + switch (shape) { + case KEY_ONLY: + return LDContext.create(key); + case STUB: + return stubContext(key); + case WIDE: { + com.launchdarkly.sdk.ContextBuilder builder = LDContext.builder(key).name("Wide"); + for (int i = 0; i < 20; i++) { + builder.set("attribute" + i, "value" + i); + } + return builder.build(); + } + case NESTED: + return LDContext.builder(key) + .name("Nested") + .set("address", LDValue.buildObject() + .put("street", "1 Main St") + .put("city", "Springfield") + .put("geo", LDValue.buildObject().put("lat", 1.5).put("lon", -2.5).build()) + .build()) + .set("tags", LDValue.buildArray() + .add("a").add("b").add("c").add("d").add("e").build()) + .build(); + case MULTI: + return LDContext.createMulti( + stubContext(key), + LDContext.builder(ContextKind.of("device"), "device-" + key) + .set("os", LDValue.buildObject() + .put("name", "Android").put("version", 14).build()) + .build()); + default: + throw new IllegalArgumentException(shape.name()); + } + } + + /** Nine attributes, matching the shape iOS calls its stub context. */ + private static LDContext stubContext(String key) { + return LDContext.builder(key) + .name("Stub Context") + .set("firstName", "Ada") + .set("lastName", "Lovelace") + .set("email", "ada@example.com") + .set("country", "US") + .set("age", 36) + .set("score", 12.5) + .set("verified", true) + .set("plan", "enterprise") + .build(); + } + + /** The privacy settings that change what redaction has to do. */ + private static final class PrivacyShape { + final String name; + final boolean allAttributesPrivate; + final Collection privateAttributes; + + PrivacyShape(String name, boolean allAttributesPrivate, Collection privateAttributes) { + this.name = name; + this.allAttributesPrivate = allAttributesPrivate; + this.privateAttributes = privateAttributes; + } + } + + private static PrivacyShape noRedaction() { + return new PrivacyShape("no redaction", false, Collections.emptyList()); + } + + private static List privacyShapes() { + return Arrays.asList( + noRedaction(), + new PrivacyShape("1 global private", false, + Collections.singletonList(AttributeRef.fromLiteral("email"))), + new PrivacyShape("all private", true, Collections.emptyList())); + } + + private OutboundEventBuffer makeBuffer(PrivacyShape privacy) { + return new OutboundEventBuffer(privacy.allAttributesPrivate, privacy.privateAttributes, + true, Integer.MAX_VALUE, logger); + } + + private static Event.FeatureRequest featureEvent(LDContext context, boolean requireFullEvent) { + return new Event.FeatureRequest(CREATION_DATE, FLAG_KEY, context, FLAG_VERSION, VARIATION, + LDValue.of(true), LDValue.of(false), null, null, requireFullEvent, null, false); + } + + private static Event.FeatureRequest summaryEvent(LDContext context, int flagIndex) { + return new Event.FeatureRequest(CREATION_DATE, FLAG_KEY + "-" + flagIndex, context, + FLAG_VERSION, VARIATION, LDValue.of(true), LDValue.of(false), null, null, false, null, + false); + } + + // MARK: Harness + + /** + * Times {@code body}, returning nanoseconds per iteration. + *

+ * Warmed first, because a JVM measured cold reports the interpreter rather than the compiled code. + * Then run several times with the fastest round reported: a garbage collection or a descheduled + * thread can only ever make a round slower, so the minimum is the closest estimate of the work + * itself. That is also why these numbers should not be read as what a busy device would see. + */ + private double measure(int iterations, IntConsumer body) { + int warmup = Math.max(10_000, iterations / 4); + for (int i = 0; i < warmup; i++) { + body.accept(i); + } + + double best = Double.MAX_VALUE; + for (int round = 0; round < ROUNDS; round++) { + long start = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + body.accept(i); + } + long elapsed = System.nanoTime() - start; + best = Math.min(best, (double) elapsed / iterations); + } + return best; + } + + /** How long one call took, across many of them, when the tail is the point. */ + private static final class Distribution { + final double median; + final double mean; + final double p999; + final double worst; + + Distribution(double median, double mean, double p999, double worst) { + this.median = median; + this.mean = mean; + this.p999 = p999; + this.worst = worst; + } + } + + /** + * Times every call rather than the enclosing loop, so the tail survives to be reported. + *

+ * One round, and the median rather than the fastest. Every other figure here reports the cheapest of + * several rounds, on the reasoning that noise can only make a round slower -- but here an outlier is the + * measurement, and that aggregation would throw away the answer. + */ + private Distribution measureDistribution(int iterations, IntConsumer body) { + int warmup = Math.max(10_000, iterations / 4); + for (int i = 0; i < warmup; i++) { + body.accept(i); + } + + long[] samples = new long[iterations]; + for (int i = 0; i < iterations; i++) { + long start = System.nanoTime(); + body.accept(i); + samples[i] = System.nanoTime() - start; + } + + long total = 0; + for (long sample : samples) { + total += sample; + } + Arrays.sort(samples); + return new Distribution(samples[iterations / 2], + (double) total / iterations, + samples[Math.min(iterations - 1, (int) ((long) iterations * 999 / 1000))], + samples[iterations - 1]); + } + + private static final class DistributionMeasurement { + final String name; + final Distribution distribution; + + DistributionMeasurement(String name, Distribution distribution) { + this.name = name; + this.distribution = distribution; + } + } + + private static final class Measurement { + final String name; + final double nanosPerOp; + + Measurement(String name, double nanosPerOp) { + this.name = name; + this.nanosPerOp = nanosPerOp; + } + } + + private static void report(String title, List results) { + int width = 0; + for (Measurement result : results) { + width = Math.max(width, result.name.length()); + } + StringBuilder out = new StringBuilder("\n").append(title).append('\n'); + for (Measurement result : results) { + out.append(" ").append(pad(result.name, width)).append(" ") + .append(format(result.nanosPerOp)).append('\n'); + } + emit(out); + } + + private static void reportDistribution(String title, List results) { + int width = 0; + for (DistributionMeasurement result : results) { + width = Math.max(width, result.name.length()); + } + StringBuilder out = new StringBuilder("\n").append(title).append('\n') + .append(" ").append(pad("", width)) + .append(" ").append(column("median")) + .append(" ").append(column("mean")) + .append(" ").append(column("p99.9")) + .append(" ").append(column("worst")).append('\n'); + for (DistributionMeasurement result : results) { + out.append(" ").append(pad(result.name, width)) + .append(" ").append(format(result.distribution.median)) + .append(" ").append(format(result.distribution.mean)) + .append(" ").append(format(result.distribution.p999)) + .append(" ").append(format(result.distribution.worst)).append('\n'); + } + emit(out); + } + + private static void emit(StringBuilder out) { + String text = out.toString(); + System.out.print(text); + System.out.flush(); + if (logFile != null) { + logFile.print(text); + logFile.flush(); + } + } + + /** Renders a count with thousands separators, so a table title reads as a quantity, not a code literal. */ + private static String count(int value) { + return String.format("%,d", value); + } + + /** Right-aligns a column heading over the fixed width {@link #format} produces. */ + private static String column(String title) { + StringBuilder padded = new StringBuilder(); + for (int i = title.length(); i < 11; i++) { + padded.append(' '); + } + return padded.append(title).toString(); + } + + private static String pad(String value, int width) { + StringBuilder padded = new StringBuilder(value); + while (padded.length() < width) { + padded.append(' '); + } + return padded.toString(); + } + + private static String format(double nanoseconds) { + if (nanoseconds >= 1_000_000) { + return String.format("%8.2f ms", nanoseconds / 1_000_000); + } + if (nanoseconds >= 1_000) { + return String.format("%8.2f \u00b5s", nanoseconds / 1_000); + } + return String.format("%8.1f ns", nanoseconds); + } +} diff --git a/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/LDClientEventTest.java b/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/LDClientEventTest.java index 544c416bd..86dd42309 100644 --- a/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/LDClientEventTest.java +++ b/launchdarkly-android-client-sdk/src/androidTest/java/com/launchdarkly/sdk/android/LDClientEventTest.java @@ -17,7 +17,9 @@ import com.launchdarkly.sdk.android.DataModel.Flag; import com.launchdarkly.sdk.android.LDConfig.Builder.AutoEnvAttributes; import com.launchdarkly.sdk.android.integrations.DedupingHook; +import com.launchdarkly.sdk.android.integrations.LDCrashHandler; import com.launchdarkly.sdk.android.integrations.Hook; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; import com.launchdarkly.sdk.android.subsystems.PersistentDataStore; import com.launchdarkly.sdk.internal.GsonHelpers; import com.launchdarkly.sdk.json.JsonSerialization; @@ -26,8 +28,14 @@ import org.junit.Test; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; import okhttp3.HttpUrl; +import okhttp3.mockwebserver.Dispatcher; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import okhttp3.mockwebserver.RecordedRequest; @@ -95,6 +103,165 @@ public void testTrackData() throws IOException, InterruptedException { } } + @Test + public void flushAndWaitReportsDelivery() throws IOException, InterruptedException { + try (MockWebServer mockEventsServer = new MockWebServer()) { + mockEventsServer.start(); + mockEventsServer.enqueue(new MockResponse()); + + LDConfig ldConfig = baseConfigBuilder(mockEventsServer).build(); + try (LDClient client = LDClient.init(application, ldConfig, ldContext, 0)) { + client.track("test-event"); + + assertTrue(client.flushAndWait(5, TimeUnit.SECONDS)); + LDValue[] events = getEventsFromLastRequest(mockEventsServer, 2); + assertCustomEvent(events[1], ldContext, "test-event"); + } + } + } + + @Test + public void crashHandlerDeliversTheEventsBeforePassingTheCrashOn() throws IOException, InterruptedException { + Thread.UncaughtExceptionHandler originalDefault = Thread.getDefaultUncaughtExceptionHandler(); + try (MockWebServer mockEventsServer = new MockWebServer()) { + mockEventsServer.start(); + mockEventsServer.enqueue(new MockResponse()); + + LDConfig ldConfig = baseConfigBuilder(mockEventsServer).build(); + try (LDClient client = LDClient.init(application, ldConfig, ldContext, 0)) { + int[] requestsSeenByNextHandler = {-1}; + Thread.setDefaultUncaughtExceptionHandler((thread, throwable) -> + requestsSeenByNextHandler[0] = mockEventsServer.getRequestCount()); + LDCrashHandler.install(5, TimeUnit.SECONDS); + client.track("test-event"); + + Thread.getDefaultUncaughtExceptionHandler() + .uncaughtException(Thread.currentThread(), new RuntimeException("boom")); + + assertEquals(1, requestsSeenByNextHandler[0]); + LDValue[] events = getEventsFromLastRequest(mockEventsServer, 2); + assertCustomEvent(events[1], ldContext, "test-event"); + } + } finally { + Thread.setDefaultUncaughtExceptionHandler(originalDefault); + } + } + + @Test + public void flushAndWaitReportsFailureOnceClosed() throws IOException { + try (MockWebServer mockEventsServer = new MockWebServer()) { + mockEventsServer.start(); + + LDConfig ldConfig = baseConfigBuilder(mockEventsServer).build(); + LDClient client = LDClient.init(application, ldConfig, ldContext, 0); + client.close(); + + assertFalse(client.flushAndWait(5, TimeUnit.SECONDS)); + } + } + + @Test + public void flushAndWaitDeliversEveryEnvironmentAtOnce() throws Exception { + // Each post is held for longer than half the budget, so the call can only report true if the + // environments' deliveries ran side by side rather than one after the other. + try (MockWebServer mockEventsServer = new MockWebServer()) { + mockEventsServer.setDispatcher(new Dispatcher() { + @Override + public MockResponse dispatch(RecordedRequest request) { + return new MockResponse().setHeadersDelay(1_500, TimeUnit.MILLISECONDS); + } + }); + mockEventsServer.start(); + + Map secondaryKeys = new HashMap<>(); + secondaryKeys.put("second", "second-mobile-key"); + LDConfig ldConfig = baseConfigBuilder(mockEventsServer) + .secondaryMobileKeys(secondaryKeys) + .build(); + try (LDClient client = LDClient.init(application, ldConfig, ldContext, 0)) { + client.track("primary-event"); + LDClient.getForMobileKey("second").track("second-event"); + + assertTrue(client.flushAndWait(2_500, TimeUnit.MILLISECONDS)); + assertEquals(2, mockEventsServer.getRequestCount()); + } + } + } + + @Test + public void flushAndWaitWithTheMostNegativeTimeoutDoesNotWait() throws IOException { + // toNanos saturates at Long.MIN_VALUE, and unclamped that wraps round to a wait for as long + // as the delivery takes -- which would then be reported as delivered. + try (MockWebServer mockEventsServer = new MockWebServer()) { + mockEventsServer.start(); + mockEventsServer.enqueue(new MockResponse().setHeadersDelay(3, TimeUnit.SECONDS)); + + LDConfig ldConfig = baseConfigBuilder(mockEventsServer).build(); + try (LDClient client = LDClient.init(application, ldConfig, ldContext, 0)) { + client.track("test-event"); + + long started = System.nanoTime(); + assertFalse(client.flushAndWait(Long.MIN_VALUE, TimeUnit.NANOSECONDS)); + long waitedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started); + assertTrue("waited " + waitedMillis + "ms", waitedMillis < 1_000); + } + } + } + + @Test + public void flushAndWaitReportsFailureWhenTheDeliveryIsCanceled() throws IOException { + // A custom event processor may hand back a future that is canceled; Future.get then throws + // CancellationException, which is unchecked and must not escape a boolean answer. + try (MockWebServer mockEventsServer = new MockWebServer()) { + mockEventsServer.start(); + + LDConfig ldConfig = baseConfigBuilder(mockEventsServer) + .events(clientContext -> new CancelingEventProcessor()) + .build(); + try (LDClient client = LDClient.init(application, ldConfig, ldContext, 0)) { + assertFalse(client.flushAndWait(5, TimeUnit.SECONDS)); + } + } + } + + /** An event processor whose deliveries are always canceled before they can report. */ + private static final class CancelingEventProcessor implements EventProcessor { + @Override + public Future flushAsync() { + FutureTask delivery = new FutureTask<>(() -> true); + delivery.cancel(false); + return delivery; + } + + @Override + public void flush() {} + + @Override + public void blockingFlush() {} + + @Override + public void setInBackground(boolean inBackground) {} + + @Override + public void setOffline(boolean offline) {} + + @Override + public void close() {} + + @Override + public void recordEvaluationEvent(LDContext context, String flagKey, int flagVersion, + int variation, LDValue value, EvaluationReason reason, + LDValue defaultValue, boolean requireFullEvent, + Long debugEventsUntilDate) {} + + @Override + public void recordIdentifyEvent(LDContext context) {} + + @Override + public void recordCustomEvent(LDContext context, String eventKey, LDValue data, + Double metricValue) {} + } + @Test public void testTrackDataValueNull() throws IOException, InterruptedException { try (MockWebServer mockEventsServer = new MockWebServer()) { diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AnalyticsEventSender.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AnalyticsEventSender.java new file mode 100644 index 000000000..09edd9a19 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AnalyticsEventSender.java @@ -0,0 +1,116 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogValues; +import com.launchdarkly.sdk.internal.events.EventSender; +import com.launchdarkly.sdk.internal.http.HttpHelpers; +import com.launchdarkly.sdk.internal.http.HttpProperties; + +import java.io.Closeable; +import java.io.IOException; +import java.net.URI; +import java.util.Date; + +import okhttp3.Headers; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/** + * Posts a batch of analytics events under a payload ID the caller chooses. + *

+ * This exists for the sake of one header. java-sdk-internal's sender invents a payload ID per call, + * which is the right thing when the events being sent only ever existed in memory: the sole reason to + * send the same events twice is its own immediate retry, and it holds the ID for that. Once events are + * kept on disk, a delivery can also be repeated by a later run of the application, after a process died + * without learning whether the service had received the batch. Those two attempts are the same delivery + * and have to say so, or the events are counted twice - and counting them twice is worse than losing + * them, because an inflated exposure count silently biases an experiment rather than visibly shrinking + * it. + *

+ * The ID belongs to the batch, which is why it is the batch's identity in {@link EventStore} rather than + * something generated at send time. + */ +final class AnalyticsEventSender implements Closeable { + private static final MediaType JSON = MediaType.parse("application/json; charset=utf-8"); + private static final String EVENT_SCHEMA_HEADER = "X-LaunchDarkly-Event-Schema"; + private static final String EVENT_SCHEMA_VERSION = "4"; + private static final String PAYLOAD_ID_HEADER = "X-LaunchDarkly-Payload-ID"; + + private final OkHttpClient client; + private final boolean ownsClient; + private final Headers baseHeaders; + private final LDLogger logger; + + AnalyticsEventSender(HttpProperties httpProperties, LDLogger logger) { + this.logger = logger; + if (httpProperties.getSharedHttpClient() == null) { + this.client = httpProperties.toHttpClientBuilder().build(); + this.ownsClient = true; + } else { + this.client = httpProperties.getSharedHttpClient(); + this.ownsClient = false; + } + this.baseHeaders = httpProperties.toHeadersBuilder().build(); + } + + /** + * Posts a batch once. The retry is the caller's, so that the wait before it does not hold a thread. + * + * @param body the JSON request body + * @param eventCount how many events the body holds, for logging + * @param payloadId identifies this delivery, the same on every attempt at it + * @param eventsBaseUri the events service + * @return whether the events arrived, whether the SDK must stop, and the service's clock; neither + * set means the failure may pass and the batch is worth trying again + */ + EventSender.Result sendBatch(byte[] body, int eventCount, String payloadId, URI eventsBaseUri) { + URI uri = HttpHelpers.concatenateUriPath(eventsBaseUri, + StandardEndpoints.ANALYTICS_EVENTS_REQUEST_PATH); + Request request = new Request.Builder() + .url(uri.toString()) + .headers(baseHeaders) + .addHeader("Content-Type", "application/json") + .addHeader(EVENT_SCHEMA_HEADER, EVENT_SCHEMA_VERSION) + .addHeader(PAYLOAD_ID_HEADER, payloadId) + .post(RequestBody.create(body, JSON)) + .build(); + + try (Response response = client.newCall(request).execute()) { + if (response.isSuccessful()) { + logger.debug("Posted {} event(s)", eventCount); + return new EventSender.Result(true, false, parseDate(response)); + } + logger.warn("Error posting events: HTTP status {}", response.code()); + if (!LDUtil.isHttpErrorRecoverable(response.code())) { + return new EventSender.Result(false, true, null); + } + } catch (IOException e) { + logger.warn("Error posting events: {}", LogValues.exceptionSummary(e)); + } + return new EventSender.Result(false, false, null); + } + + /** + * The service's own clock, which the SDK trusts over the device's when deciding whether a debug + * window has closed. + */ + private Date parseDate(Response response) { + try { + return response.headers().getDate("Date"); + } catch (RuntimeException e) { + return null; + } + } + + @Override + public void close() { + if (!ownsClient) { + return; + } + client.dispatcher().executorService().shutdown(); + client.connectionPool().evictAll(); + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AndroidPlatformState.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AndroidPlatformState.java index 357648917..3fb8c39e7 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AndroidPlatformState.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/AndroidPlatformState.java @@ -18,8 +18,13 @@ import android.os.Bundle; import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogValues; +import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; +import java.io.InputStreamReader; +import java.nio.charset.Charset; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; @@ -169,6 +174,40 @@ public File getCacheDir() { return application.getCacheDir(); } + @Override + public File getNoBackupFilesDir() { + return application.getNoBackupFilesDir(); + } + + @Override + public String getProcessName() { + // Returned as the platform gives it, punctuation and all. Whatever needs a name safe for some + // other purpose is responsible for making it so, because only that caller knows what "safe" + // means there; see EventStore. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + return Application.getProcessName(); + } + // Before API 28 there is no supported way to ask, so the process reads its own command line, + // which is where the platform gets the answer from as well. + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(new FileInputStream("/proc/self/cmdline"), Charset.forName("UTF-8")))) { + String line = reader.readLine(); + if (line != null) { + // The command line is NUL-terminated, and Java keeps those bytes as characters. + int end = line.indexOf(0); + String name = end < 0 ? line : line.substring(0, end); + if (!name.trim().isEmpty()) { + return name.trim(); + } + } + } catch (Exception e) { + logger.debug("Could not determine the process name: {}", LogValues.exceptionSummary(e)); + } + // A wrong answer here would let two processes share one file, so the fallback is the pid, which + // is unique among live processes even though it does not survive a restart. + return "pid-" + android.os.Process.myPid(); + } + @Override public void close() { connectivityChangeListeners.clear(); diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/ComponentsImpl.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/ComponentsImpl.java index 8c9cc0454..5652bcb88 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/ComponentsImpl.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/ComponentsImpl.java @@ -4,6 +4,7 @@ import com.launchdarkly.sdk.EvaluationReason; import com.launchdarkly.sdk.LDContext; import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.integrations.EventPersistence; import com.launchdarkly.sdk.android.integrations.EventProcessorBuilder; import com.launchdarkly.sdk.android.integrations.HooksConfigurationBuilder; import com.launchdarkly.sdk.android.integrations.HttpConfigurationBuilder; @@ -20,14 +21,12 @@ import com.launchdarkly.sdk.android.subsystems.HookConfiguration; import com.launchdarkly.sdk.android.subsystems.HttpConfiguration; import com.launchdarkly.sdk.android.subsystems.PluginsConfiguration; -import com.launchdarkly.sdk.internal.events.DefaultEventProcessor; import com.launchdarkly.sdk.internal.events.DefaultEventSender; -import com.launchdarkly.sdk.internal.events.Event; -import com.launchdarkly.sdk.internal.events.EventsConfiguration; +import com.launchdarkly.sdk.internal.events.EventSender; -import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.Future; /** * This class contains the package-private implementations of component factories and builders whose @@ -75,6 +74,12 @@ public void flush() {} @Override public void blockingFlush() {} + @Override + public Future flushAsync() { + // Nothing was recorded, so there is nothing undelivered to warn the caller about. + return new LDSuccessFuture<>(true); + } + @Override public void setInBackground(boolean inBackground) {} @@ -101,33 +106,54 @@ static final class EventProcessorBuilderImpl extends EventProcessorBuilder @Override public EventProcessor build(ClientContext clientContext) { ClientContextImpl clientContextImpl = ClientContextImpl.get(clientContext); - EventsConfiguration eventsConfig = new EventsConfiguration( - allAttributesPrivate, - capacity, - null, // contextDeduplicator - not needed for client-side use - diagnosticRecordingIntervalMillis, - clientContextImpl.getDiagnosticStore(), - new DefaultEventSender( - LDUtil.makeHttpProperties(clientContext), - StandardEndpoints.ANALYTICS_EVENTS_REQUEST_PATH, - StandardEndpoints.DIAGNOSTIC_EVENTS_REQUEST_PATH, - 0L, // use default retry delay - false, // disable gzip compression for Android + EventSender diagnosticEventSender = new DefaultEventSender( + LDUtil.makeHttpProperties(clientContext), + StandardEndpoints.ANALYTICS_EVENTS_REQUEST_PATH, + StandardEndpoints.DIAGNOSTIC_EVENTS_REQUEST_PATH, + 0L, // use default retry delay + false, // disable gzip compression for Android + clientContext.getBaseLogger()); + return new DirectEventProcessor( + new OutboundEventBuffer( + allAttributesPrivate, + privateAttributes, + true, // perContextSummarization - enable for client SDK + capacity, + clientContext.getBaseLogger()), + makeEventStore(clientContext, clientContextImpl), + diagnosticEventSender, + new AnalyticsEventSender(LDUtil.makeHttpProperties(clientContext), clientContext.getBaseLogger()), - 1, // eventSendingThreadPoolSize clientContext.getServiceEndpoints().getEventsBaseUri(), + clientContextImpl.getDiagnosticStore(), + capacity, + eventPersistence == EventPersistence.IMMEDIATE, flushIntervalMillis, + diagnosticRecordingIntervalMillis, + DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, clientContext.isInBackground(), true, // initiallyOffline - privateAttributes, - true // perContextSummarization - enable for client SDK - ); - return new DefaultEventProcessorWrapper(new DefaultEventProcessor( - eventsConfig, EventUtil.makeEventsTaskExecutor(), - Thread.NORM_PRIORITY, // note, we may want to make this configurable as it is in java-server-sdk + EventUtil.makeEventsCommitExecutor(), + EventUtil.makeDiagnosticsTaskExecutor(), clientContext.getBaseLogger() - )); + ); + } + + /** + * Builds the store this environment's events are kept in until LaunchDarkly has them. + *

+ * The mobile key and the process are both part of where it writes, so an application configured + * for several environments, or running the SDK in several processes, gets one store per + * combination and they never touch each other's events. + */ + private EventStore makeEventStore(ClientContext clientContext, ClientContextImpl impl) { + return EventStore.create( + impl.getPlatformState(), + clientContext.getMobileKey(), + capacity, + eventPersistence != EventPersistence.DISABLED, + clientContext.getBaseLogger()); } @Override @@ -136,76 +162,9 @@ public LDValue describeConfiguration(ClientContext clientContext) { .put("allAttributesPrivate", allAttributesPrivate) .put("diagnosticRecordingIntervalMillis", diagnosticRecordingIntervalMillis) .put("eventsCapacity", capacity) - .put("diagnosticRecordingIntervalMillis", diagnosticRecordingIntervalMillis) .put("eventsFlushIntervalMillis", flushIntervalMillis) .build(); } - - /** - * Adapter from the public component interface of EventProcessor to the internal - * implementation class from java-sdk-internal. - */ - private final class DefaultEventProcessorWrapper implements EventProcessor { - private final DefaultEventProcessor eventProcessor; - - DefaultEventProcessorWrapper(DefaultEventProcessor eventProcessor) { - this.eventProcessor = eventProcessor; - } - - @Override - public void recordEvaluationEvent( - LDContext context, - String flagKey, - int flagVersion, - int variation, - LDValue value, - EvaluationReason reason, - LDValue defaultValue, - boolean requireFullEvent, - Long debugEventsUntilDate - ) { - eventProcessor.sendEvent(new Event.FeatureRequest( - System.currentTimeMillis(), flagKey, context, flagVersion, variation, - value, defaultValue, reason, null, requireFullEvent, - debugEventsUntilDate, false)); - } - - @Override - public void recordIdentifyEvent(LDContext context) { - eventProcessor.sendEvent(new Event.Identify(System.currentTimeMillis(), context)); - } - - @Override - public void recordCustomEvent(LDContext context, String eventKey, LDValue data, Double metricValue) { - eventProcessor.sendEvent(new Event.Custom(System.currentTimeMillis(), eventKey, - context, data, metricValue)); - } - - @Override - public void setInBackground(boolean inBackground) { - eventProcessor.setInBackground(inBackground); - } - - @Override - public void setOffline(boolean offline) { - eventProcessor.setOffline(offline); - } - - @Override - public void flush() { - eventProcessor.flushAsync(); - } - - @Override - public void blockingFlush() { - eventProcessor.flushBlocking(); - } - - @Override - public void close() throws IOException { - eventProcessor.close(); - } - } } static final class HttpConfigurationBuilderImpl extends HttpConfigurationBuilder diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DirectEventProcessor.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DirectEventProcessor.java new file mode 100644 index 000000000..baef19a85 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DirectEventProcessor.java @@ -0,0 +1,1565 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogValues; +import com.launchdarkly.sdk.EvaluationReason; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; +import com.launchdarkly.sdk.internal.events.DiagnosticEvent; +import com.launchdarkly.sdk.internal.events.DiagnosticStore; +import com.launchdarkly.sdk.internal.events.Event; +import com.launchdarkly.sdk.internal.events.EventSender; +import com.launchdarkly.sdk.internal.events.EventSummarizer; +import com.launchdarkly.sdk.internal.events.Sampler; + +import java.io.Closeable; +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * The Android SDK's analytics event processor. + *

+ * Recording an event summarizes it immediately and, only if it has to be delivered in full, holds it. A full + * event is kept as an {@link Event} until a commit turns the run of them into bytes and writes them, so + * evaluations of untracked flags cost a counter increment and evaluations of tracked ones cost that plus a + * list append. Nothing is encoded on the thread that evaluated. + *

+ * Where the line falls is the subject of O12 in the event-encoding research, and it falls between the events + * an application asked for by name and the ones it did not. {@code track} and {@code identify} are commit + * points, because an application reporting an error is saying this matters more than the microseconds it + * costs and the crash it describes may be moments away; a commit encodes the whole held run, so the + * exposures leading up to the error go down with it. Evaluations get no such promise and are committed once + * {@link #PENDING_COMMIT_THRESHOLD} of them have accumulated where persistence is on, on {@code flush}, or + * when the application flushes from its own crash handler. + *

+ * Whether a commit point runs on the caller's thread is the application's choice, through + * {@link com.launchdarkly.sdk.android.integrations.EventProcessorBuilder#eventPersistence( + * com.launchdarkly.sdk.android.integrations.EventPersistence)}. Only at + * {@code IMMEDIATE} can {@code track} promise the event is on disk by the time it returns. + */ +final class DirectEventProcessor implements EventProcessor { + /** + * How many full events may be held unencoded before one of them pays for a commit. + *

+ * It counts events rather than bytes because there are no bytes until the commit runs. Each commit is one + * write of everything it encoded, so this, with the commit points, is what decides how large a write is. + * It sets two things at once. + *

+ * The first is how many evaluations a termination can take. That is narrower than everything a crash + * could take: an uncaught exception runs the application's handler while the process is still alive, and + * a handler that flushes commits the whole held run, so this is the window for the terminations that run + * nothing on the way out -- {@code SIGKILL}, a native crash, an ANR kill, and the system reclaiming a + * backgrounded process. A {@code track} closes it too, because it commits. + *

+ * The second is the worst a {@code track} can cost, since it encodes whatever is held before returning. + * The common case is far below the bound, because the commit executor keeps the run drained; raising + * this trades that tail against the number of writes. + */ + private static final int PENDING_COMMIT_THRESHOLD = 32; + + /** + * How long {@link #close()} spends waiting for the final delivery before it gives up and returns. + *

+ * {@code close()} runs on the caller's thread, which for an application shutting down is usually the + * main one, so this has to stay well inside the five seconds Android allows before an unanswered + * input event becomes an ANR. Two seconds is where the return on waiting longer falls off: the + * events client keeps pooled connections for only five seconds against a thirty second flush + * interval, so this post nearly always pays a full DNS, TCP and TLS handshake -- about three round + * trips, which two seconds covers up to roughly a 600ms RTT. A network slower than that is one the + * post is likely to fail on anyway. + *

+ * Overshooting the budget is cheaper than it looks, because the delivery is not canceled when the + * budget expires; see {@link #close()}. + */ + static final long DEFAULT_CLOSE_BUDGET_MILLIS = 2_000; + + /** How long a delivery that failed in a way that may pass waits for its one retry, as the other SDKs do. */ + static final long RETRY_DELAY_MILLIS = 1_000; + + // Set in the constructor and never changed. + + private final EventStore store; + private final EventSender diagnosticEventSender; + private final AnalyticsEventSender analyticsEventSender; + private final URI eventsUri; + private final DiagnosticStore diagnosticStore; + private final long flushIntervalMillis; + private final long diagnosticRecordingIntervalMillis; + private final long closeBudgetMillis; + private final ScheduledExecutorService scheduler; + /** + * Where a commit no caller waits for runs: a thread of its own, not {@link #scheduler}. + *

+ * The scheduler also delivers, and one delivery can hold it for as long as two network timeouts. A + * commit queued behind that would leave a {@code DEFERRED} {@code track} in memory for all of it, and + * let the pending run fill toward capacity meanwhile. + */ + private final ExecutorService commitExecutor; + private final ExecutorService diagnosticExecutor; + private final LDLogger logger; + + /** + * How many events the SDK will hold in total, across {@link #pending} and the store. + */ + private final int capacity; + + /** + * Whether a commit point encodes and writes before returning, rather than queueing that work. + */ + private final boolean commitOnCallerThread; + + // Guarded by commitLock. + + /** + * Held for the whole of a commit, so that only one runs at a time. + *

+ * This is what {@code track}'s guarantee rests on. Without it a commit already in flight could take the + * caller's event out of {@link #pending} before the caller got there, leaving the caller nothing to + * write and returning while those bytes were still being produced somewhere else. Waiting here instead + * means that when the call returns the event is on disk, whichever commit put it there. + *

+ * Distinct from {@link #recordLock}, which is only held long enough to hand the run and the counters + * over: encoding under this lock must not block a thread that is merely recording. + */ + private final Object commitLock = new Object(); + + /** + * The last recording a finished commit covered. Written under {@link #commitLock}, and read outside it + * so that a caller whose recording is already durable does not queue for the lock at all. + *

+ * This is what makes concurrent {@code IMMEDIATE} commit points a group commit. The first caller's + * commit takes every recording made before it took the run, including those of callers now waiting + * behind it; they find their recordings covered and return, and the next caller still uncovered + * commits everything that arrived in the meantime. Without it each waiter committed again in turn, + * and each of those commits wrote a summary of its own. + */ + private volatile long committedSequence; + + // Guarded by recordLock. + + /** + * Guards everything one recording writes: {@link #pending} and the summary counters behind + * {@link #eventBuffer}. + *

+ * One evaluation can produce a counter, a full event and a debug event, and the three have to + * land together. Taken separately, a flush landing between them splits one evaluation across two + * payloads, and a {@link #close()} landing between them delivers the counter and then refuses the + * full event -- leaving a summary that says an evaluation happened and no event to go with it. + *

+ * Held for the appends and the handover of the run, never across the encode. Recording runs on + * whichever thread evaluated a flag, which on Android is usually the main one. The worst it may + * wait for is another thread's memory operation; if the encoder ran under this lock, an + * evaluation would instead wait on the dominant cost of the whole path. + */ + private final Object recordLock = new Object(); + + /** + * Full events recorded but not yet encoded, guarded by {@link #recordLock}. + *

+ * Held rather than serialized because serializing early would not make them durable: the store stages + * bytes into memory too, and only {@link EventStore#commit()} reaches the file. Both forms are equally + * lost to a crash, so the encode may as well happen where it is cheapest. + */ + private final List pending = new ArrayList<>(); + + /** + * Numbers every recording, guarded by {@link #recordLock}, so a caller at a commit point can tell whether + * a commit has already covered it. + */ + private long recordedSequence; + + /** + * Its summary counters are guarded by {@link #recordLock}. Its serializer is reached only under + * {@link #commitLock}. + */ + private final OutboundEventBuffer eventBuffer; + + // Guarded by stateLock. + + /** + * Guards {@link #flushTask} and {@link #diagnosticTask}, and every change to {@link #inBackground} + * and {@link #offline}, so the tasks are rescheduled in the order the changes happen. Those two + * are atomic so they can be read without it. + */ + private final Object stateLock = new Object(); + private ScheduledFuture flushTask; + private ScheduledFuture diagnosticTask; + private final AtomicBoolean inBackground; + private final AtomicBoolean offline; + + // Guarded by submitLock. + + /** + * Guards the handover from running to shut down: held for the length of a submit, and by + * {@link #close()} while it queues the release of each thread's resources and stops the + * executors accepting work. Nothing blocking happens under it. + */ + private final Object submitLock = new Object(); + + /** Set under {@link #submitLock} once close() has queued the release of those resources. */ + private boolean shuttingDown = false; + + /** + * The retry queued after a delivery failed in a way that may pass, guarded by {@link #submitLock}. + * One at a time: it retries every batch still waiting, so a second would only repeat it. + */ + private ScheduledFuture pendingRetry; + + // Guarded by flushLock. + + /** + * Guards {@link #pendingFlush} and {@link #pendingFlushAnswersACaller}. Taken on a caller's + * thread and on the delivery thread, never while holding {@link #recordLock}, and nothing + * blocking happens under it. + */ + private final Object flushLock = new Object(); + + /** + * The delivery that is queued but has not started, which a flush request arriving now can wait + * on instead of queueing another. Null while nothing is queued, and cleared again as the queued + * delivery begins, which is the point past which it can no longer speak for what is recorded. + */ + private LDAwaitFuture pendingFlush; + + /** + * Whether a {@link #flushAsync()} caller, who is there to hear the outcome, is waiting on + * {@link #pendingFlush}, as opposed to only callers that discard it. + */ + private boolean pendingFlushAnswersACaller; + + // Atomic or volatile, so read and written without a lock. + + private final AtomicBoolean closed = new AtomicBoolean(false); + // Set when the service tells us to stop, e.g. because the mobile key is invalid. + private volatile boolean disabled = false; + private volatile boolean diagnosticInitSent = false; + private final AtomicLong lastKnownPastTime = new AtomicLong(0); + /** + * Whether a previous run of the application left events behind. + *

+ * Those events are already as old as whatever happened to that run, so they are sent as soon as the + * SDK is allowed to send anything rather than at the next interval. The application in front of the + * user has moved on; an error report about a crash is not worth much thirty seconds late, and the + * process might not live that long either. + */ + private final AtomicBoolean hasEventsFromPreviousRun = new AtomicBoolean(false); + + /** + * Whether a commit is already queued, so a run of recordings past the threshold submits one task rather + * than one per event. + */ + private final AtomicBoolean commitScheduled = new AtomicBoolean(false); + + private final AtomicBoolean capacityExceeded = new AtomicBoolean(false); + + /** Whether the summarizer being full has already been reported, so it is logged once per run. */ + private final AtomicBoolean summaryContextsExceeded = new AtomicBoolean(false); + + private final AtomicLong droppedEvents = new AtomicLong(0); + + /** + * True while a diagnostic event is on its way to the service, so that a later one is dropped + * rather than queued behind it. + */ + private final AtomicBoolean diagnosticPostInFlight = new AtomicBoolean(false); + + /** + * How many events the SDK accepted and then dropped for good. + *

+ * A batch the service refuses in a way that may pass is kept and retried, so it is not a loss. + * What is lost is what the store never received: an event or a summary that could not be + * serialized when it was committed, or that serialized too large to store. A delivery that + * finds nothing to send cannot tell those apart from events that arrived, so without this it + * would report success for them. + *

+ * A count rather than a flag because a delivery can answer long after it ran: one that met a + * retryable failure answers only when its retry does, and another delivery can come and go in + * between. Each delivery takes the count as it closes its batch and answers only for the losses + * up to that mark, so it is never told about a loss that happened after it had taken everything + * it could speak for. Only the order of two values ever matters, never the difference between + * them, so a failure that lost several events need only move it once. + *

+ * Atomic because commits run on the commit executor and on callers' threads, and a caller + * waiting behind a retry is answered from a pooled thread. + */ + private final AtomicLong lostEvents = new AtomicLong(0); + + /** + * The mark up to which a loss has already been reported to a {@link #flushAsync()} caller, so + * that no loss is reported twice and none goes unreported. + */ + private final AtomicLong reportedLostEvents = new AtomicLong(0); + + DirectEventProcessor( + OutboundEventBuffer eventBuffer, + EventStore store, + EventSender diagnosticEventSender, + AnalyticsEventSender analyticsEventSender, + URI eventsUri, + DiagnosticStore diagnosticStore, + int capacity, + boolean commitOnCallerThread, + long flushIntervalMillis, + long diagnosticRecordingIntervalMillis, + long closeBudgetMillis, + boolean initiallyInBackground, + boolean initiallyOffline, + ScheduledExecutorService scheduler, + ExecutorService commitExecutor, + ExecutorService diagnosticExecutor, + LDLogger logger + ) { + this.eventBuffer = eventBuffer; + this.store = store; + this.commitOnCallerThread = commitOnCallerThread; + this.diagnosticEventSender = diagnosticEventSender; + this.analyticsEventSender = analyticsEventSender; + this.eventsUri = eventsUri; + this.diagnosticStore = diagnosticStore; + this.capacity = capacity; + this.flushIntervalMillis = flushIntervalMillis; + this.diagnosticRecordingIntervalMillis = diagnosticRecordingIntervalMillis; + this.closeBudgetMillis = closeBudgetMillis; + this.scheduler = scheduler; + this.commitExecutor = commitExecutor; + this.diagnosticExecutor = diagnosticExecutor; + this.logger = logger; + this.inBackground = new AtomicBoolean(initiallyInBackground); + this.offline = new AtomicBoolean(initiallyOffline); + + // Off the calling thread: this is on the path of LDClient.init, and it reads the filesystem. + submit(() -> { + store.recoverInterruptedLog(); + if (store.pendingBatches().isEmpty()) { + return; + } + // Raised before offline is read, and claimed by whichever side sees both: going online writes + // offline and then claims the flag, so at least one of the two finds the other's write, and + // the claim makes it exactly one. Reading offline first left a window where going online + // found no flag yet and this then found the SDK still offline, and neither sent them. + hasEventsFromPreviousRun.set(true); + if (!offline.get() && !isStopped() && hasEventsFromPreviousRun.compareAndSet(true, false)) { + deliverPayload(); + } + }); + + synchronized (stateLock) { + updateScheduledTasks(initiallyInBackground, initiallyOffline); + } + } + + @Override + public void recordEvaluationEvent( + LDContext context, + String flagKey, + int flagVersion, + int variation, + LDValue value, + EvaluationReason reason, + LDValue defaultValue, + boolean requireFullEvent, + Long debugEventsUntilDate + ) { + try { + if (isStopped() || context == null) { + return; + } + long timestamp = System.currentTimeMillis(); + boolean debug = shouldDebugEvent(debugEventsUntilDate); + // Only built when something is kept beyond the counter. Most flags are not tracked, and for + // those an evaluation is a counter increment with nothing left behind for the collector. + // Built before the lock is taken, so that the critical section is only the writes. + Event.FeatureRequest event = requireFullEvent || debug + ? new Event.FeatureRequest(timestamp, flagKey, context, flagVersion, variation, value, + defaultValue, reason, null, requireFullEvent, debugEventsUntilDate, false) + : null; + Event debugEvent = debug ? event.toDebugEvent() : null; + boolean contextsExceeded; + boolean warnContextsExceeded; + boolean needsCommit; + synchronized (recordLock) { + if (closed.get()) { + return; + } + recordedSequence++; + contextsExceeded = !eventBuffer.summarize(timestamp, flagKey, flagVersion, variation, + value, defaultValue, context); + // Claimed under the lock that the commit resets it under, so the warning belongs to the + // run whose summarizer turned this evaluation away rather than to the next one. + warnContextsExceeded = contextsExceeded + && summaryContextsExceeded.compareAndSet(false, true); + if (requireFullEvent) { + addPending(event); + } + if (debugEvent != null) { + addPending(debugEvent); + } + needsCommit = pending.size() >= PENDING_COMMIT_THRESHOLD; + } + if (contextsExceeded) { + reportContextsExceeded(warnContextsExceeded); + } + // Deliberately no commit point. An evaluation is expected to cost what a map lookup costs, and it + // is usually the main thread doing it; the store writes these on its own thread once enough of them + // have piled up, and the next event recorded at a commit point makes them durable along with itself. + if (needsCommit) { + scheduleCommitWherePersisting(); + } + } catch (RuntimeException e) { + // This runs on the application's thread, usually inside a flag evaluation, and an + // analytics failure must not become the application's failure. Errors such as + // OutOfMemoryError are left to propagate, so the application's crash reporting sees them. + logUnexpectedError(e); + } + } + + @Override + public void recordIdentifyEvent(LDContext context) { + if (isStopped() || context == null) { + return; + } + commitAtCommitPoint(record(new Event.Identify(System.currentTimeMillis(), context))); + } + + @Override + public void recordCustomEvent(LDContext context, String eventKey, LDValue data, Double metricValue) { + if (isStopped() || context == null) { + return; + } + commitAtCommitPoint(record(new Event.Custom(System.currentTimeMillis(), eventKey, context, data, + metricValue))); + } + + /** + * Holds an event for the next commit to encode, counting it as dropped if the SDK is already full. + *

+ * Capacity is consulted before anything is encoded, so an event that will not be kept is never encoded. + * That ordering is what bounds an application re-evaluating a tracked flag in a render loop: once the + * limit is reached the cost of an evaluation falls back to its summary counter, however fast the loop runs. + * + * @return the recording's sequence number, for the commit point that follows it; 0 if it was refused + */ + long record(Event event) { + try { + long sequence; + boolean needsCommit; + synchronized (recordLock) { + // The close check that decides the outcome, as against the fast path the public record methods + // take before building the event. The commit lifts the run out under this same lock, so testing + // the flag here orders a record against close()'s final commit: either the event is in the list + // before that commit takes it, or it is refused. Tested outside the lock the two interleave, and + // an event can be left in a list that nothing will drain again. + if (closed.get()) { + return 0; + } + sequence = ++recordedSequence; + addPending(event); + needsCommit = pending.size() >= PENDING_COMMIT_THRESHOLD; + } + if (needsCommit) { + scheduleCommitWherePersisting(); + } + return sequence; + } catch (RuntimeException e) { + // As in recordEvaluationEvent: on the caller's thread, so a failure is logged, not thrown. + logUnexpectedError(e); + return 0; + } + } + + /** + * Commits at a commit point, on the caller's thread or off it as the application asked. + *

+ * Committing on the caller's thread is what lets {@code track} promise its event is on disk by the time + * it returns. Scheduling it instead keeps the encode and the write off that thread, and the event is + * durable a moment later rather than immediately -- which is nothing at all when persistence is off, + * since there is no disk for an early commit to reach. + * + * @param through the last recording the caller needs durable: its own, or for a flush everything so far + */ + private void commitAtCommitPoint(long through) { + try { + if (commitOnCallerThread) { + commitDurablyThrough(through); + } else { + scheduleCommitWherePersisting(); + } + } catch (RuntimeException e) { + // As in recordEvaluationEvent: every commit point is on the caller's thread. + logUnexpectedError(e); + } + } + + /** + * Commits unless a finished commit already covers recording {@code through}, checked again once the lock + * is held, since the commit a caller waited behind is the one most likely to have covered it. + */ + private void commitDurablyThrough(long through) { + if (committedSequence >= through) { + return; + } + synchronized (commitLock) { + if (committedSequence >= through) { + return; + } + commitDurably(); + } + } + + /** @return the sequence number of the last recording so far */ + private long lastRecordedSequence() { + synchronized (recordLock) { + return recordedSequence; + } + } + + /** + * Queues the commit a full pending run or a commit point asks for, where there is a disk for it to reach. + *

+ * Without persistence a commit makes nothing durable. All it would do is move the encode into the + * middle of the application's evaluations, where it competes with them for the CPU and splits their + * counters across summaries; left alone, the run waits for the flush, which encodes it anyway, and + * capacity still bounds how much is held. + */ + private void scheduleCommitWherePersisting() { + if (store.isPersisting()) { + scheduleCommit(); + } + } + + /** Queues a commit unless one is already queued, so a run of recordings asks for one rather than many. */ + private void scheduleCommit() { + if (commitScheduled.compareAndSet(false, true)) { + try { + commitExecutor.execute(guarded(this::runScheduledCommit)); + } catch (RuntimeException e) { // shut down by close(), whose own commit takes these events + commitScheduled.set(false); + } + } + } + + /** + * Clears the scheduling flag before committing, not after, so events recorded while this runs can queue + * a commit of their own rather than waiting for the next one to be triggered. + *

+ * Does nothing once closed. {@link #close()} sets the flag before its final commit, which takes + * everything recorded up to then, and nothing is recorded after it; a commit reaching here later could + * only run against a store that has been released. + */ + private void runScheduledCommit() { + commitScheduled.set(false); + if (closed.get()) { + return; + } + commitDurably(); + } + + /** + * Encodes everything recorded since the last commit and makes it survive the process. + *

+ * This is the only place events are serialized. It runs on the commit executor when the pending run + * reaches {@link #PENDING_COMMIT_THRESHOLD}, and on the caller's thread for {@code flush}, where the + * caller has asked to wait for exactly this. + *

+ * The evaluations counted since the last commit are written too, because a summary is only worth having + * if it covers the evaluations that led up to whatever is about to happen. + */ + private void commitDurably() { + synchronized (commitLock) { + List run; + List summaries; + long through; + // Both taken at once, so that an evaluation's counter and its full event are staged by the + // same commit. Taken separately, a commit landing between the two writes one evaluation makes + // stages the counter and leaves the event for the next one -- or, at close, for none at all. + synchronized (recordLock) { + through = recordedSequence; + run = pending.isEmpty() ? Collections.emptyList() : new ArrayList<>(pending); + pending.clear(); + summaries = eventBuffer.takeSummaries(); + summaryContextsExceeded.set(false); + // Reserved in the same critical section the run leaves pending in, so capacity counts + // it in one place or the other throughout the encode. Each staged event uses up its + // reservation as it arrives, so none is counted twice either. + store.reserve(run.size()); + } + try { + stageRun(run); + } finally { + store.releaseReservations(); + } + stageSummaries(summaries); + if (eventBuffer.takeSerializationFailure()) { + lostEvents.incrementAndGet(); + } + store.commit(); + committedSequence = through; + } + } + + /** + * Encodes a run of events as one run and stages the bytes. + *

+ * The run was taken under {@link #recordLock} and is encoded outside it, so recording does not wait on + * the encoder. Staging bypasses capacity because the decision to keep these events was already made in + * {@link #record}, and refusing them here would drop events the SDK has counted as accepted; the + * reservation taken with the run is what keeps them counted until they are staged. + *

+ * Requires {@link #commitLock}: two threads staging separate runs would stage them in whichever order + * they finished encoding, which is not the order they were recorded in. + */ + private void stageRun(List run) { + for (byte[] serialized : eventBuffer.serializeAll(run)) { + if (!store.stageReserved(serialized)) { + reportRefusedByStore("event", serialized); + } + } + } + + /** + * Turns the evaluation counters already taken into summary events in the store. + */ + private void stageSummaries(List summaries) { + for (byte[] summary : eventBuffer.serializeSummaries(summaries)) { + // Bypassing capacity: a summary is not a new event, it is the record of evaluations already + // counted, and dropping it would lose all of them at once. + if (!store.stage(summary, true)) { + reportRefusedByStore("summary event", summary); + } + } + } + + /** + * Records the loss of something the store would not take. Capacity is bypassed on both paths, so + * what is left is a frame too large to store, which no later attempt can fix. + */ + private void reportRefusedByStore(String what, byte[] serialized) { + logger.error("Dropping {} of {} bytes, too large to store", what, serialized.length); + lostEvents.incrementAndGet(); + } + + /** + * Claims the losses up to a mark on behalf of one {@link #flushAsync()} caller. + * + * @param mark the loss count as of the close of that caller's batch + * @return true if a loss at or before the mark had not yet been reported to anyone, in which + * case this caller is the one reporting it + */ + private boolean takeEventsLost(long mark) { + // A loop rather than getAndUpdate, which this SDK cannot use below API 24. + while (true) { + long reported = reportedLostEvents.get(); + if (mark <= reported) { + return false; + } + if (reportedLostEvents.compareAndSet(reported, mark)) { + return true; + } + } + } + + /** + * Holds one event for the next flush, unless it was sampled out or there is no room. Requires + * {@link #recordLock}. + */ + private void addPending(Event event) { + // Ahead of the capacity check, because an event the SDK was never going to send is not a + // loss and must not be counted as one. Sampling and capacity are different reasons not to + // keep an event, and only the second is one the SDK owes anyone a count of. + if (!Sampler.shouldSample(event.getSamplingRatio())) { + return; + } + if (pending.size() + store.getPendingEventCount() >= capacity) { + if (capacityExceeded.compareAndSet(false, true)) { + logger.warn("Exceeded event queue capacity. Increase capacity to avoid dropping events."); + } + droppedEvents.incrementAndGet(); + return; + } + capacityExceeded.set(false); + pending.add(event); + } + + /** + * Counts an evaluation the summarizer turned away, which it does when counting it would have meant + * holding a context beyond the configured capacity. + *

+ * A refused evaluation is a loss in the same sense a refused event is -- nothing later reconstructs + * a counter -- so it is reported the same way, through the dropped count diagnostics carry. + */ + private void reportContextsExceeded(boolean warn) { + // Warned once per delivery run rather than once per process: the summarizer's contexts are + // cleared with each run, so a later overflow is a new one worth hearing about. + if (warn) { + logger.warn("Exceeded the number of contexts that can be summarized at once." + + " Increase capacity to avoid dropping evaluations."); + } + droppedEvents.incrementAndGet(); + } + + /** + * @return the number of events dropped for capacity since this was last called, counting + * evaluations the summarizer turned away + */ + long getAndClearDroppedCount() { + return droppedEvents.getAndSet(0); + } + + @Override + public void setInBackground(boolean inBackground) { + synchronized (stateLock) { + if (this.inBackground.getAndSet(inBackground) == inBackground) { + return; + } + updateScheduledTasks(inBackground, offline.get()); + } + } + + @Override + public void setOffline(boolean offline) { + synchronized (stateLock) { + if (this.offline.getAndSet(offline) == offline) { + return; + } + updateScheduledTasks(inBackground.get(), offline); + } + } + + @Override + public void flush() { + if (isStopped()) { + return; + } + commitBeforeDelivery(); + queueDelivery(false); + } + + @Override + public void blockingFlush() { + if (isStopped()) { + return; + } + commitBeforeDelivery(); + Future delivery = queueDelivery(false); + try { + delivery.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + logUnexpectedError(e.getCause() == null ? e : e.getCause()); + } + } + + @Override + public Future flushAsync() { + if (isStopped()) { + return new LDSuccessFuture<>(false); + } + commitBeforeDelivery(); + return queueDelivery(true); + } + + /** + * The commit point every flush starts with, before anything is handed to the events thread. + *

+ * Written on the caller's thread wherever events persist, not only where the application asked for + * that as it does for {@code track} and {@code identify}. A flush is what an application calls when it + * is about to end the process on purpose, and a write it only queued is one the process may end + * before. Where nothing persists, a commit makes nothing durable, and is left to the delivery. + *

+ * It does not wait for the events thread, which can be held by a post or by a retry long after a + * caller with a deadline has stopped waiting. The delivery commits again when it runs, but that is + * too late to be the only commit: {@code flushAndWait} exists for the caller that is about to be + * killed, and a timed-out wait must still leave the events durable. + */ + private void commitBeforeDelivery() { + try { + if (commitOnCallerThread || store.isPersisting()) { + commitDurablyThrough(lastRecordedSequence()); + } + } catch (RuntimeException e) { + // As in recordEvaluationEvent: every commit point is on the caller's thread. + logUnexpectedError(e); + } + } + + /** + * Queues a delivery, or hands back one that is already queued and has not started. + *

+ * A delivery that has not started yet will send everything recorded before it starts. That + * includes the caller's events, so waiting on it is as good as queuing a new one. Without this, + * flushes arriving faster than a post completes each queue their own, and the one that matters -- + * the {@code flushAndWait} at shutdown -- waits behind all of them. + * + * @param answersACaller true if the caller will hear the outcome, so that the delivery reports + * any events lost up to the batch it closes, and false if the caller discards it + */ + private Future queueDelivery(boolean answersACaller) { + synchronized (flushLock) { + if (pendingFlush != null) { + pendingFlushAnswersACaller |= answersACaller; + return pendingFlush; + } + LDAwaitFuture result = new LDAwaitFuture<>(); + if (submit(() -> runDelivery(result)) == null) { + // Shutting down, so there is no thread left to deliver on and nothing will be sent. + return new LDSuccessFuture<>(false); + } + pendingFlush = result; + pendingFlushAnswersACaller = answersACaller; + return result; + } + } + + /** + * Runs one delivery on behalf of every flush request that joined it, and tells them all how it + * went. + *

+ * A caller who will hear the answer is told no if events were dropped for good up to the batch + * this delivery closed and nobody has been told yet, as well as if the delivery fails: a + * delivery that finds nothing to send is not evidence that everything recorded arrived. + */ + private void runDelivery(LDAwaitFuture result) { + boolean answersACaller = false; + synchronized (flushLock) { + // Requests arriving from here on need a delivery of their own: this one is about to take + // the buffer, and what it takes is all it can speak for. + if (pendingFlush == result) { + pendingFlush = null; + answersACaller = pendingFlushAnswersACaller; + pendingFlushAnswersACaller = false; + } + } + DeliveryOutcome outcome; + try { + outcome = commitAndDeliverReportingOutcome(); + } catch (Throwable t) { + // Caught here rather than left to guarded(), because a caller is waiting on the future + // and completing it matters more than the stack reaching the executor. + logUnexpectedError(t); + outcome = DeliveryOutcome.CLOSED_NOTHING; + } + if (outcome.retry == null) { + answer(result, outcome.delivered, answersACaller, outcome.lostEventsMark); + } else { + completeWhenRetryEnds(result, outcome, answersACaller); + } + } + + /** + * Answers the waiters when the queued retry does, rather than telling them a batch failed when + * it is about to be sent again. Waiting happens on a pooled thread because the retry runs a + * second from now on this one, which must be free to take it. + */ + private void completeWhenRetryEnds(LDAwaitFuture result, DeliveryOutcome outcome, + boolean answersACaller) { + LDAwaitFuture retried = LDFutures.fromFuture(outcome.retry); + retried.addListener(() -> { + boolean delivered = false; + try { + delivered = Boolean.TRUE.equals(retried.get()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + // Reported as not delivered. + } + // The mark the delivery took, not one taken now: a loss during the wait belongs to + // whichever delivery closes the batch after it, not to this one. + answer(result, delivered, answersACaller, outcome.lostEventsMark); + }); + } + + /** + * Completes a delivery's future. A caller who will hear the answer is told no if this delivery + * failed, or if events the SDK accepted were lost for good up to the mark it took and no caller + * has been told so yet. + */ + private void answer(LDAwaitFuture result, boolean delivered, boolean answersACaller, + long lostEventsMark) { + if (answersACaller) { + delivered &= !takeEventsLost(lostEventsMark); + } + result.set(delivered); + } + + @Override + public void close() throws IOException { + if (!closed.compareAndSet(false, true)) { + return; + } + synchronized (stateLock) { + updateScheduledTasks(inBackground.get(), offline.get()); + } + // One budget for the whole of close(), which runs on the caller's thread -- for an application + // shutting down, usually the main one -- so it has to stay well inside the five seconds Android + // allows before an unanswered input event becomes an ANR. The budget bounds the caller, not + // the work: whatever outlasts it is left to finish on its own thread. + long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(closeBudgetMillis); + + // Staging and writing come first, and get the budget first. A caller who closed the client and + // then let the process end has to find those events on the next run, and that is worth more + // than the post below: a batch on disk is delivered next run, while an event never staged is + // gone whether or not the post would have succeeded. + Future finalCommit = commitWithinBudget(deadlineNanos); + + // Then deliver what is buffered, with whatever is left. This waits rather than firing and + // forgetting because it is this run's last chance to send them. While offline that chance is + // not taken: offline is the application telling the SDK to stay off the network, and shutting + // down does not revoke that. The commit above has written those events down where persistence + // is on, so they go out on a later run; where it is off they are discarded. + Callable finalDelivery = this::deliverPayloadAndRetryLater; + Future delivery = submit(finalDelivery); + if (delivery != null) { + try { + awaitDelivery(delivery, Math.max(0, deadlineNanos - System.nanoTime())); + } catch (TimeoutException e) { + // Deliberately not canceled. The batch is closed and on its way, so interrupting now + // would make the loss certain, while leaving it to run costs nothing: the scheduler + // thread is a daemon, and returning from close() does not end an Android process. + logger.warn("Gave up waiting for the final event delivery after {}ms;" + + " it continues in the background", closeBudgetMillis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + logUnexpectedError(e.getCause() == null ? e : e.getCause()); + } + } + // The store and the senders, by contrast, are released on whichever thread uses them, after + // whatever is still in flight there. Closing them inline would pull them out from under a + // delivery, or a diagnostic post, that we just decided not to wait for. shutdown() then + // refuses new work while letting what is already queued finish -- unlike shutdownNow(), which + // would interrupt those and strand the futures of anything it discarded. + // + // Queueing and shutting down are one step under submitLock, so that a flush cannot land in + // between them and put a delivery behind the release, where it would find the store closed. + synchronized (submitLock) { + shuttingDown = true; + if (pendingRetry != null) { + // Not interrupted if it is already posting: it checks shuttingDown before it starts, and + // one that got past that runs ahead of the release queued below on the same thread. + pendingRetry.cancel(false); + pendingRetry = null; + } + // The store is released only once the final commit is through with it, which may be after + // the budget if the disk stalled. That waits here, on the scheduler thread, and never on the + // caller's. + try { + scheduler.submit(guarded(() -> { + awaitQuietly(finalCommit); + releaseDeliveryResources(); + })); + } catch (RuntimeException e) { // the executor was shut down under us + releaseDeliveryResources(); + } + queueRelease(diagnosticExecutor, this::releaseDiagnosticResources); + scheduler.shutdown(); + diagnosticExecutor.shutdown(); + // Commits already queued there still run, and find the processor closed. + commitExecutor.shutdown(); + } + } + + /** + * Commits everything recorded so far for close(), waiting for it no longer than the budget allows. + *

+ * On the commit executor rather than the caller's thread, because each step of a commit can block + * for as long as the device takes: {@link #commitLock} may be held by a commit or a delivery that + * stalled on the disk, the store's I/O lock by the write it stalled on, and the write this one + * makes is disk I/O too. Committing on the caller's thread was bounded by none of them, so a + * stalled flash write turned into an ANR on the main thread. + *

+ * A commit that outlasts the budget is left to finish, for the same reason as the delivery: + * the executor thread is a daemon, returning from close() does not end the process, and a write + * cut off partway leaves a torn frame that the next run reads past, not events lost. + * + * @return the commit, for the store's release to wait on, or null if it could not be queued + */ + private Future commitWithinBudget(long deadlineNanos) { + Future commit; + try { + commit = commitExecutor.submit(guarded(this::commitDurably)); + } catch (RuntimeException e) { // shut down under us; the final delivery still commits first + return null; + } + try { + commit.get(Math.max(0, deadlineNanos - System.nanoTime()), TimeUnit.NANOSECONDS); + } catch (TimeoutException e) { + logger.warn("Gave up waiting for the final event commit after {}ms;" + + " it continues in the background", closeBudgetMillis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + logUnexpectedError(e.getCause() == null ? e : e.getCause()); + } + return commit; + } + + /** Waits for a task to finish however it ends. Only for threads that are not the caller's. */ + private static void awaitQuietly(Future task) { + if (task == null) { + return; + } + try { + task.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + // Logged where it ran. + } + } + + /** Queues a release on the thread that owns those resources, or runs it here if that thread has gone. */ + private void queueRelease(ExecutorService executor, Runnable release) { + try { + executor.submit(guarded(release)); + } catch (RuntimeException e) { // the executor was shut down under us + release.run(); // so nothing can still be using them + } + } + + /** What the delivery thread owns: the store it drains and the sender it posts batches through. */ + private void releaseDeliveryResources() { + store.close(); + closeQuietly(analyticsEventSender); + } + + /** What the diagnostics thread owns: the sender nothing else posts through. */ + private void releaseDiagnosticResources() { + closeQuietly(diagnosticEventSender); + } + + private void closeQuietly(Closeable closeable) { + try { + closeable.close(); + } catch (IOException e) { + logUnexpectedError(e); + } + } + + /** + * Serializes and sends everything buffered, for the periodic flush, which has nobody waiting to + * find out how it went. It is a fixed-delay series, so a run is only ever scheduled once the one + * before it has finished and these cannot pile up the way requested flushes could. + */ + private void deliverPayload() { + deliverPayloadAndRetryLater(); + } + + /** + * Writes everything accepted so far, then delivers. + *

+ * This commit looks redundant with the one in {@link #commitAndCloseBatch()}, and online it is: the + * second finds nothing pending and returns. It is kept for the case where it is the only one. A + * delivery refused for being offline or disabled returns before {@code commitAndCloseBatch}, so + * without this a flush while offline would leave everything recorded in memory, and a process that + * then died would lose it -- the case persistence exists for. + */ + private DeliveryOutcome commitAndDeliverReportingOutcome() { + commitDurably(); + return deliverPayloadAndRetryLater(); + } + + /** + * How a delivery went, or the retry that will say, for a caller waiting to find out. + */ + private static final class DeliveryOutcome { + /** For a delivery that closed no batch, so there are no losses it can speak for. */ + static final DeliveryOutcome CLOSED_NOTHING = new DeliveryOutcome(false, null, 0); + + final boolean delivered; + /** Non-null when a batch failed in a way that may pass, and a retry of every batch is queued. */ + final Future retry; + /** + * The loss count as of the close of this delivery's batch, which is as far as it can answer + * for; see {@link #takeEventsLost(long)}. + */ + final long lostEventsMark; + + private DeliveryOutcome(boolean delivered, Future retry, long lostEventsMark) { + this.delivered = delivered; + this.retry = retry; + this.lostEventsMark = lostEventsMark; + } + } + + /** + * Waits for a delivery and, when it queued a retry, for that too, within one budget. + * + * @param timeoutNanos how long to wait in all, or a negative number to wait as long as it takes + * @return true if the events reached the service, or if there were none to send + */ + private boolean awaitDelivery(Future delivery, long timeoutNanos) + throws InterruptedException, ExecutionException, TimeoutException { + long deadline = System.nanoTime() + timeoutNanos; + DeliveryOutcome outcome = timeoutNanos < 0 ? delivery.get() + : delivery.get(timeoutNanos, TimeUnit.NANOSECONDS); + if (outcome == null) { + return false; + } + if (outcome.retry == null) { + return outcome.delivered; + } + Boolean retried = timeoutNanos < 0 ? outcome.retry.get() + : outcome.retry.get(Math.max(0, deadline - System.nanoTime()), TimeUnit.NANOSECONDS); + return Boolean.TRUE.equals(retried); + } + + /** + * Closes what has been committed into a batch and delivers every batch waiting, queueing one retry + * if a batch failed in a way that may pass. + *

+ * Runs on the scheduler thread for a requested or periodic delivery, and on the caller's thread + * at close. Deliveries can therefore overlap: a retry runs a second behind the delivery that + * queued it, and the one at close does not wait for the scheduler. Each closes a batch of its + * own under {@link #commitLock}, so no event is committed into two batches, and a batch that two + * of them do both post carries the same payload ID both times, which LaunchDarkly discards + * rather than counting the events twice. + */ + private DeliveryOutcome deliverPayloadAndRetryLater() { + if (disabled || offline.get()) { + return DeliveryOutcome.CLOSED_NOTHING; + } + + long lostEventsMark = commitAndCloseBatch(); + + BatchesOutcome outcome = deliverPendingBatches(); + if (outcome == BatchesOutcome.RETRYABLE) { + Future retry = scheduleRetry(); + return new DeliveryOutcome(false, retry, lostEventsMark); + } + return new DeliveryOutcome(outcome == BatchesOutcome.DELIVERED, null, lostEventsMark); + } + + /** + * Commits everything accepted so far and closes it into a batch to deliver, taking the loss + * count as of that close. + *

+ * Both under {@link #commitLock}, for two reasons. A commit landing between the two would stage + * events into the log this batch is about to close, which splits what one commit staged across + * two payloads -- an evaluation's counter in this batch and its event in the next. And the mark + * has to be the count as of the close, because that is what this delivery can answer for: a loss + * after it is one whose events were never in this batch. + *

+ * A requested flush has usually just committed in {@link #commitAndDeliverReportingOutcome()}, so this + * commit often finds nothing to do. It cannot be the one left out: the periodic delivery, the one + * going online starts, the one recovery starts, and the final one at close all reach here without + * that earlier commit. + * + * @return the loss count as of the close, for {@link #takeEventsLost(long)} + */ + private long commitAndCloseBatch() { + synchronized (commitLock) { + commitDurably(); + store.closeBatch(); + return lostEvents.get(); + } + } + + private enum BatchesOutcome { DELIVERED, FAILED, RETRYABLE } + + /** + * Makes one attempt at every batch waiting, oldest first: the others are deliveries an earlier attempt + * did not finish, or that a previous run of the application never got to start. + *

+ * Stops at the first failure that may pass. The batches behind it would most likely meet the same + * network, and the retry takes all of them again. + */ + private BatchesOutcome deliverPendingBatches() { + boolean allDelivered = true; + for (EventStore.Batch batch : store.pendingBatches()) { + if (disabled || offline.get()) { + return BatchesOutcome.FAILED; + } + BatchesOutcome outcome = deliver(batch); + if (outcome == BatchesOutcome.RETRYABLE) { + return outcome; + } + allDelivered &= outcome == BatchesOutcome.DELIVERED; + } + return allDelivered ? BatchesOutcome.DELIVERED : BatchesOutcome.FAILED; + } + + /** + * Queues the retry on the scheduler rather than sleeping for it there, which would hold every flush, + * and close()'s final delivery, behind the wait. + * + * @return the retry, a retry already queued, or null if the processor is shutting down + */ + private Future scheduleRetry() { + synchronized (submitLock) { + if (shuttingDown) { + return null; + } + if (pendingRetry != null && !pendingRetry.isDone()) { + return pendingRetry; + } + try { + pendingRetry = scheduler.schedule(this::retryPendingBatches, RETRY_DELAY_MILLIS, + TimeUnit.MILLISECONDS); + } catch (RuntimeException e) { // the executor was shut down under us + return null; + } + logger.warn("Will retry posting events in {}ms after a failure", RETRY_DELAY_MILLIS); + return pendingRetry; + } + } + + /** + * The one retry. A failure here is not retried again; the batches wait on disk, or in memory, for the + * next flush. + */ + private boolean retryPendingBatches() { + synchronized (submitLock) { + if (shuttingDown) { + // close() has queued the release of the store and the sender behind this. + return false; + } + } + if (disabled || offline.get()) { + return false; + } + try { + return deliverPendingBatches() == BatchesOutcome.DELIVERED; + } catch (RuntimeException e) { + logUnexpectedError(e); + return false; + } + } + + /** + * @return DELIVERED if the batch is no longer the SDK's problem, whether because it arrived or because + * it never can; RETRYABLE if it failed in a way that may pass + */ + private BatchesOutcome deliver(EventStore.Batch batch) { + byte[] body; + try { + body = store.body(batch); + } catch (EventStore.UnreadableBatchException e) { + // Lost like a refused batch: nothing will ever send it, so it is removed, the rest are still + // attempted, and the caller hears that this pass lost events. + logger.error("Dropping {} stored event(s) this version cannot read", batch.eventCount); + store.remove(batch); + return BatchesOutcome.FAILED; + } catch (IOException e) { + // Kept: the file is still there, and a read that failed may not fail next time. + logger.warn("Could not read stored events: {}", LogValues.exceptionSummary(e)); + return BatchesOutcome.RETRYABLE; + } + if (body == null) { + // Already delivered by another process of this application, or holding no events. Either way + // nothing is lost and there is nothing to keep. + store.remove(batch); + return BatchesOutcome.DELIVERED; + } + if (diagnosticStore != null) { + diagnosticStore.recordEventsInBatch(batch.eventCount); + } + store.startedSending(batch); + try { + EventSender.Result result = analyticsEventSender.sendBatch(body, batch.eventCount, + batch.payloadId, eventsUri); + handleResponse(result); + if (result != null && (result.isSuccess() || result.isMustShutDown())) { + // Forgotten once the service has either taken the events or told us to stop sending + // them. Anything else - a timeout, a 503, no network - leaves the batch where it is, to + // be tried again by the retry, the next flush or the next run of the application. + store.remove(batch); + return result.isSuccess() ? BatchesOutcome.DELIVERED : BatchesOutcome.FAILED; + } + return result == null ? BatchesOutcome.FAILED : BatchesOutcome.RETRYABLE; + } catch (Exception e) { + logUnexpectedError(e); + return BatchesOutcome.FAILED; + } finally { + // Counted again unless the batch was removed above. + store.stoppedSending(batch); + } + } + + /** + * Posts an event that has already been built. Requires the caller to have checked + * {@link #diagnosticsSuspended()}, because for the periodic event building it is what consumes + * the period. + */ + private void sendDiagnosticEvent(DiagnosticEvent diagnosticEvent, boolean isInit) { + try { + byte[] data = diagnosticEvent.getJsonValue().toJsonString() + .getBytes(StandardCharsets.UTF_8); + handleResponse(diagnosticEventSender.sendDiagnosticEvent(data, eventsUri)); + if (isInit) { + // Attempted, not delivered. A failed post gives back an unsuccessful Result rather + // than throwing, so this marks the init event done either way and the process never + // retries it. That matches DefaultEventProcessor, which is the behaviour to keep: + // diagnostics are best-effort telemetry about the SDK, and a retry that outlived + // its own init would describe a configuration the application has moved on from. + diagnosticInitSent = true; + } + } catch (Exception e) { + logUnexpectedError(e); + } + } + + private void sendDiagnosticStats() { + if (diagnosticsSuspended() || diagnosticStore == null || !claimDiagnosticPost()) { + return; + } + DiagnosticStore store = diagnosticStore; + postDiagnostic(() -> { + // The check that decides the outcome, and it has to come before the event is built rather + // than after. createEventAndReset hands back the period and clears it -- the stream inits, + // the events-in-batch count and the period start all move -- and getAndClearDroppedCount + // does the same for the dropped count. This runs on the diagnostics thread, which may have + // been busy with an earlier post for as long as that post took, so the state can easily + // have changed since the checks above. Bailing out after the event was built would discard + // a period outright and leave the next event describing a window that begins after the + // reset; bailing out here leaves everything where it is, for the next period to carry. + if (diagnosticsSuspended()) { + return; + } + sendDiagnosticEvent(store.createEventAndReset(getAndClearDroppedCount(), 0), false); + }); + } + + /** + * Takes the diagnostic posting thread, or reports that the last event is still on it. + *

+ * A new event is dropped rather than queued behind the old one. Diagnostics are best-effort + * telemetry, a post can take tens of seconds against a network that never answers, and queueing + * would let an outage accumulate events describing an SDK state the application has since moved + * past -- the same reasoning that makes a failed init event final rather than retried. + */ + private boolean claimDiagnosticPost() { + if (diagnosticPostInFlight.compareAndSet(false, true)) { + return true; + } + logger.debug("Skipped a diagnostic event because the previous one is still being posted"); + return false; + } + + /** + * Hands a claimed post to the diagnostics thread, releasing the claim once it ends. + * + * @param post what to run there, which must hold a claim from {@link #claimDiagnosticPost()} + */ + private void postDiagnostic(Runnable post) { + Runnable releasing = () -> { + try { + post.run(); + } finally { + diagnosticPostInFlight.set(false); + } + }; + try { + diagnosticExecutor.submit(guarded(releasing)); + } catch (RuntimeException e) { // the executor was shut down under us + diagnosticPostInFlight.set(false); + } + } + + /** + * Unlike analytics events, diagnostics are not sent while offline or in the background. + *

+ * {@link #updateScheduledTasks} cancels the periodic task when either becomes true, but that is + * not enough on its own. Canceling does not stop a run already underway, and the init event is + * submitted before it reaches the executor. Either can arrive here after the state changed. + */ + private boolean diagnosticsSuspended() { + return isStopped() || offline.get() || inBackground.get(); + } + + private void handleResponse(EventSender.Result result) { + if (result == null) { + return; + } + if (result.getTimeFromServer() != null) { + recordPastTime(result.getTimeFromServer().getTime()); + } + if (result.isMustShutDown()) { + disabled = true; + } + } + + /** + * Moves the threshold forwards only. Analytics and diagnostic responses are handled on separate + * threads, so a plain assignment would let an older reading of the service clock overwrite a + * newer one and keep debug events alive past the date the service set. A loop rather than + * {@code accumulateAndGet}, which needs API 24. + */ + private void recordPastTime(long timeFromServer) { + long known; + do { + known = lastKnownPastTime.get(); + if (timeFromServer <= known) { + return; + } + } while (!lastKnownPastTime.compareAndSet(known, timeFromServer)); + } + + /** + * A debug event is emitted until the date the service gave us passes. We compare against the + * last date we know to be in the past according to the service as well as the device clock, so + * that a device whose clock is wrong errs on the side of stopping sooner. + */ + private boolean shouldDebugEvent(Long debugEventsUntilDate) { + if (debugEventsUntilDate == null || debugEventsUntilDate <= 0) { + return false; + } + return debugEventsUntilDate > lastKnownPastTime.get() + && debugEventsUntilDate > System.currentTimeMillis(); + } + + /** + * Must be called holding {@code stateLock}. Once closed, this only ever cancels: close() sets the + * flag and then calls this under the same lock, so a call that got here first has its tasks + * canceled by close(), and any call after it finds the flag set. + */ + private void updateScheduledTasks(boolean inBackground, boolean offline) { + boolean stopped = closed.get(); + // Flushing stays scheduled even while we are offline or in the background; a run while + // offline returns without doing anything. Canceling it for an outage would restart the + // interval on every reconnect, and a run of brief outages would then hold events back for + // far longer than one interval. Left running, what an outage buffered goes out at the first + // run after it ends. + flushTask = enableOrDisableTask(!stopped, flushTask, flushIntervalMillis, + this::deliverPayload); + // Events a previous run left on disk have already waited out a whole process, so they do not + // wait out an interval as well. An application that dies within one interval of every start + // would otherwise never deliver them. + if (!stopped && !offline && hasEventsFromPreviousRun.compareAndSet(true, false)) { + submit(this::deliverPayload); + } + boolean diagnosticsEnabled = !stopped && diagnosticStore != null && !offline && !inBackground; + diagnosticTask = enableOrDisableTask(diagnosticsEnabled, diagnosticTask, + diagnosticRecordingIntervalMillis, this::sendDiagnosticStats); + if (diagnosticsEnabled && !diagnosticInitSent && claimDiagnosticPost()) { + DiagnosticStore store = diagnosticStore; + postDiagnostic(() -> { + // Re-checked on the posting thread: going online and coming to the foreground are + // two separate calls, and both want to send the init event we never got to send. + // Suspension is re-checked for the same reason it is for the periodic event, and + // costs nothing here: getInitEvent consumes nothing, so a skipped init is simply + // built again the next time diagnostics are enabled. + if (!diagnosticInitSent && !diagnosticsSuspended()) { + sendDiagnosticEvent(store.getInitEvent(), true); + } + }); + } + } + + private ScheduledFuture enableOrDisableTask( + boolean shouldEnable, + ScheduledFuture currentTask, + long intervalMillis, + Runnable task + ) { + if (!shouldEnable) { + if (currentTask != null) { + currentTask.cancel(false); + } + return null; + } + if (currentTask != null && !currentTask.isDone()) { + return currentTask; + } + if (currentTask != null) { + // Backstop for a throwable that escaped guarded() anyway, such as one thrown while + // logging the first: the executor marks the repeating future done and never fires it + // again, and holding that future here would make every later enable a no-op. + currentTask.cancel(false); + } + try { + // Fixed delay rather than fixed rate: a cached process stops running its tasks without + // stopping the clock, so at a fixed rate it would come back owing every run it missed and + // fire them one after another. + return scheduler.scheduleWithFixedDelay(guarded(task), intervalMillis, intervalMillis, + TimeUnit.MILLISECONDS); + } catch (RuntimeException e) { // the executor was shut down under us + return null; + } + } + + /** + * Puts work on the delivery thread, and is the one place that decides whether there is still a + * thread willing to take it. The {@code isStopped} tests the callers make first are a fast path + * that saves the work of getting here, not a guarantee; this is what a delivery is actually + * ordered against close(). + * + * @return the submitted task, or null if the processor is shutting down or already has + */ + private Future submit(Runnable task) { + synchronized (submitLock) { + if (shuttingDown) { + // close() has already queued the release of the sender. Work accepted now would be + // behind it in the queue and would run against a sender that had been closed. + return null; + } + try { + return scheduler.submit(guarded(task)); + } catch (RuntimeException e) { // the executor was shut down under us + return null; + } + } + } + + /** + * As {@link #submit(Runnable)}, for the final delivery at close(), whose outcome close() waits + * for. Not wrapped in {@link #guarded}, because here the caller is there to receive what + * escapes. + * + * @return the submitted task, or null if the processor is shutting down or already has + */ + private Future submit(Callable task) { + synchronized (submitLock) { + if (shuttingDown) { + return null; + } + try { + return scheduler.submit(task); + } catch (RuntimeException e) { // the executor was shut down under us + return null; + } + } + } + + /** + * Keeps an unexpected failure from killing a repeating task or bubbling out of the executor. + * Anything that escapes a run suppresses the rest of a {@code scheduleWithFixedDelay} series, + * so this catches {@code Throwable} and not just {@code Exception}: a + * {@code StackOverflowError} from nested {@code LDValue} data or an {@code OutOfMemoryError} + * growing the payload stream would otherwise stop flushing for the life of the process with + * nothing logged. + */ + private Runnable guarded(Runnable task) { + return () -> { + try { + task.run(); + } catch (Throwable t) { + logUnexpectedError(t); + } + }; + } + + private boolean isStopped() { + return closed.get() || disabled; + } + + private void logUnexpectedError(Throwable e) { + logger.error("Unexpected error in event processor: {}", LogValues.exceptionSummary(e)); + logger.debug("{}", LogValues.exceptionTrace(e)); + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DiskEventLog.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DiskEventLog.java new file mode 100644 index 000000000..155c67536 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DiskEventLog.java @@ -0,0 +1,181 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogValues; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +/** + * The file of events on disk this process appends to, not to be confused with diagnostic logging. + *

+ * One per process rather than one per directory, so that only one process ever writes a given log and + * recovery only ever touches its own. With one shared log, a second process starting up would close + * off, as if a previous run had left it, a log the first is still appending to through its open stream, + * and the first's later events would land in a batch already sent or deleted. Another process's log is + * left for that process to recover the next time it runs. + *

+ * Belongs to an {@link EventStore}, which makes every call holding its {@code ioLock}. The lock is handed + * in so that each call can check it is held, rather than this taking a lock of its own that would have + * to be ordered against the store's. + */ +final class DiskEventLog { + /** Names a log a single process appends to; the rest of the name is that process. */ + static final String PREFIX = "open-"; + + /** How the log is opened, so a test can make a write fail partway through a session, as a full disk does. */ + interface Opener { + FileOutputStream openForAppending(File log) throws IOException; + } + + private final File directory; + private final File file; + private final Object ioLock; + private final Opener opener; + private final LDLogger logger; + // Guarded by ioLock. + private FileOutputStream output; + + DiskEventLog(File directory, String processName, Object ioLock, Opener opener, LDLogger logger) { + this.directory = directory; + this.file = new File(directory, PREFIX + nameFor(processName)); + this.ioLock = ioLock; + this.opener = opener; + this.logger = logger; + } + + /** + * Names the log belonging to one process. + *

+ * The readable part is the process name with anything awkward in a filename replaced, which is what + * makes a directory of these diagnosable by a human. That reduction is not one-to-one, though: + * {@code com.app:remote} and {@code com.app.remote} are both legal process names and both come out as + * {@code com_app_remote}. Two processes sharing one log is the single thing this name exists to + * prevent, so the digest of the original name is appended to guarantee they never do. + */ + static String nameFor(String processName) { + String name = processName == null ? "" : processName.trim(); + if (name.isEmpty()) { + name = "unknown"; + } + StringBuilder safe = new StringBuilder(name.length() + 9); + for (int i = 0; i < name.length(); i++) { + char c = name.charAt(i); + safe.append(Character.isLetterOrDigit(c) || c == '-' || c == '_' ? c : '_'); + } + return safe.append('-').append(EventDirectory.hexDigest(name, 4)).toString(); + } + + /** Where the log is, whether or not it exists yet. */ + File file() { + return file; + } + + /** Requires {@code ioLock}. */ + boolean isOpen() { + assert Thread.holdsLock(ioLock); + return output != null; + } + + /** Requires {@code ioLock}. */ + boolean exists() { + assert Thread.holdsLock(ioLock); + return file.exists(); + } + + /** + * Appends to the log, opening it first where it is not open. + *

+ * Requires {@code ioLock}. + * + * @throws IOException if the log could not be opened or written to; none of {@code bytes} is then left + * in the log, and the log is closed, so the next append starts from a fresh open + */ + void append(byte[] bytes) throws IOException { + assert Thread.holdsLock(ioLock); + FileOutputStream stream = outputForAppending(); + long start = -1; + try { + start = stream.getChannel().size(); + stream.write(bytes); + } catch (IOException | RuntimeException e) { + // A write that fails partway can leave whole frames behind. The caller keeps every one of + // the bytes in memory to deliver from there, so frames left here as well would be sent + // twice: beside the memory copy when the log is closed this session, or by recovery on the + // next launch. + if (start >= 0) { + cutBack(stream, start); + } + closeOutput(); + throw e; + } + } + + /** Cuts the log back to {@code length}, where a failed write left it longer. */ + private void cutBack(FileOutputStream stream, long length) { + try { + stream.getChannel().truncate(length); + } catch (IOException | RuntimeException e) { + logger.debug("Could not cut back a failed write to the event log: {}", + LogValues.exceptionSummary(e)); + } + } + + /** + * Moves the log to {@code destination}, so that the next append starts a new one. The stream is closed + * first, whether or not the move succeeds. + *

+ * Requires {@code ioLock}. + * + * @return whether the log was moved + */ + boolean closeMovingTo(File destination) { + assert Thread.holdsLock(ioLock); + closeOutput(); + return file.renameTo(destination); + } + + /** Closes the stream, leaving the log where it is. Requires {@code ioLock}. */ + void closeOutput() { + assert Thread.holdsLock(ioLock); + if (output == null) { + return; + } + try { + output.close(); + } catch (IOException e) { + logger.debug("Could not close the event log: {}", LogValues.exceptionSummary(e)); + } + output = null; + } + + /** Requires {@code ioLock}. */ + private FileOutputStream outputForAppending() throws IOException { + if (output != null) { + return output; + } + if (!directory.exists() && !directory.mkdirs() && !directory.isDirectory()) { + throw new IOException("could not create " + directory); + } + boolean isNew = !file.exists() || file.length() == 0; + // Append mode is what makes each write land at the end of the file as one step, so that a + // process cannot splice its bytes into the middle of what another wrote. + FileOutputStream stream = opener.openForAppending(file); + if (isNew) { + try { + stream.write(EventLogFormat.fileHeader()); + } catch (IOException | RuntimeException e) { + // Not yet in output, so closeOutput would not reach it. + try { + stream.close(); + } catch (IOException ignored) { + // The write already failed; that is the error worth reporting. + } + throw e; + } + } + output = stream; + return output; + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventDirectory.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventDirectory.java new file mode 100644 index 000000000..2333cc42d --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventDirectory.java @@ -0,0 +1,600 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogValues; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +/** + * The directory one store keeps its events in: the log this process appends to, and beside it the batches + * that have been closed off for delivery. + *

+ * Everything about the filesystem lives here - where the directory is, what the files in it are called, and + * how they are listed, read, moved and deleted - so that {@link EventStore} is left with the buffer, the + * capacity arithmetic, and the batches it holds in memory, none of which need a disk at all. A store the + * application told to keep events off the disk has no {@code EventDirectory}, rather than one it is careful + * never to use. + *

+ * Where the directory is, is worked out on first use rather than at construction, so that a store which never + * reaches for the disk makes no filesystem call at all, not even to find out where it would have written. + * That matters because the store is built on the thread that called {@code LDClient.init}, usually the main + * one: the platform creates the no-backup directory the first time it is asked for it, and before API 28 the + * process name is read from {@code /proc}. + *

+ * Belongs to an {@link EventStore}, which makes every call holding its {@code ioLock}, exactly as + * {@link DiskEventLog} belongs to this. The lock is handed down so that each call can check it is held, + * rather than each level taking a lock of its own that would have to be ordered against the others. + */ +class EventDirectory { + /** + * Names a log closed off for delivery. The rest of the name is the batch's payload ID and how many + * events are in it, {@code ready--}. + *

+ * The count is in the name so that listing a batch this process did not close costs a directory entry + * rather than a read of the whole file. That is what every batch is at startup: the ones a previous run + * left behind, and the ones belonging to another process sharing the environment. Reading them all is + * done holding {@code ioLock}, which a commit at a commit point then waits behind. + *

+ * A batch is never appended to once it is closed, so the count is as good as the file is. + */ + private static final String BATCH_PREFIX = "ready-"; + private static final String DIRECTORY_NAME = "com.launchdarkly.events"; + + /** Where one store keeps its files. */ + interface Location { + File directory(); + + String processName(); + } + + /** How a stored file is read, so a test can make a read fail while the file is intact. */ + interface Reader { + InputStream openForReading(File file) throws IOException; + } + + private final Location location; + private final Object ioLock; + private final LDLogger logger; + private final Reader reader; + private final DiskEventLog.Opener opener; + + // Guarded by ioLock, and null until locateHoldingIoLock has run. + private File directory; + private DiskEventLog diskLog; + + /** Whether a log left by a previous run has been dealt with. Guarded by ioLock. */ + private boolean recoveryDone; + + /** + * What this process knows about each batch in the directory, by payload ID, which is how everything + * outside this class names one. + *

+ * The file is remembered rather than composed from the payload ID, because the count is part of the name + * and a batch closed by a version that did not write one has no count to compose. The count is + * remembered so that such a batch is read at most once. + *

+ * Only to be used while holding {@code ioLock}. + */ + private final Map stored = new HashMap<>(); + + /** A batch in the directory: the file it is, and how many events it holds. */ + private static final class Stored { + final File file; + final int events; + + Stored(File file, int events) { + this.file = file; + this.events = events; + } + } + + /** + * Batches LaunchDarkly has accepted whose file the filesystem would not let go of. + *

+ * Left out of every listing, because a batch that is delivered and merely still on the disk would otherwise + * be found again by the next delivery, and the one after that, and sent each time. The delete is tried once + * more on each listing, so a directory that stops being read-only clears itself. + *

+ * Only for this session: a later run, which may well be able to delete the file, treats it as it would any + * other batch a previous run left behind. LaunchDarkly discards a repeated payload ID, so the cost of that is + * a wasted request rather than events counted twice. + *

+ * Only to be used while holding {@code ioLock}. + */ + private final Set deliveredButUndeleted = new HashSet<>(); + + EventDirectory(Location location, Object ioLock, LDLogger logger, Reader reader, + DiskEventLog.Opener opener) { + this.location = location; + this.ioLock = ioLock; + this.logger = logger; + this.reader = reader; + this.opener = opener; + } + + /** Requires {@code ioLock}. */ + private void locateHoldingIoLock() { + assert Thread.holdsLock(ioLock); + if (directory == null) { + File resolved = location.directory(); + diskLog = new DiskEventLog(resolved, location.processName(), ioLock, opener, logger); + directory = resolved; + } + } + + /** + * Where the batches are kept, worked out the first time the directory is reached for. + *

+ * Requires {@code ioLock}. + */ + File directory() { + locateHoldingIoLock(); + return directory; + } + + /** Requires {@code ioLock}. */ + private DiskEventLog diskLog() { + locateHoldingIoLock(); + return diskLog; + } + + /** + * A batch is identified by its payload ID rather than by a path, so that a batch listed from the + * directory and the same batch as it was closed are one thing. + *

+ * Requires {@code ioLock}. + * + * @param events what the caller holding the batch knows it to contain, which is what names it where + * this process has neither closed nor listed it - as when it delivers a batch another process closed + */ + private File batchFile(String payloadId, int events) { + Stored known = stored.get(payloadId); + return known == null ? newBatchFile(payloadId, events) : known.file; + } + + /** Requires {@code ioLock}. */ + private File newBatchFile(String payloadId, int events) { + return new File(directory(), BATCH_PREFIX + payloadId + "-" + events); + } + + /** + * A batch as its filename gives it: everything between the prefix and the count is the payload ID. + * + * @return the count the name ends with, or -1 where it has none, which is how a version that did not + * write one named its batches. A payload ID cannot be mistaken for a count: it is a UUID, whose + * last group is twelve characters. Twelve digits are more than an {@code int} holds unless they + * start with a zero, which a count never does. + */ + private static int eventsInName(String fileName) { + int split = fileName.lastIndexOf('-'); + if (split < BATCH_PREFIX.length()) { + return -1; + } + String suffix = fileName.substring(split + 1); + if (suffix.isEmpty() || (suffix.length() > 1 && suffix.charAt(0) == '0')) { + return -1; + } + for (int i = 0; i < suffix.length(); i++) { + char c = suffix.charAt(i); + if (c < '0' || c > '9') { + return -1; + } + } + try { + return Integer.parseInt(suffix); + } catch (NumberFormatException e) { + return -1; + } + } + + /** The payload ID a filename carries, given what {@link #eventsInName} made of it. */ + private static String payloadIdInName(String fileName, int events) { + return events < 0 + ? fileName.substring(BATCH_PREFIX.length()) + : fileName.substring(BATCH_PREFIX.length(), fileName.lastIndexOf('-')); + } + + // MARK: writing + + /** What an append did. */ + static final class Append { + /** How many events a recovery closed off into batches of their own on the way. */ + final int recovered; + /** + * Whether every byte landed. False means none of them is in the log, and that the caller should + * give up on the disk. + */ + final boolean written; + + Append(int recovered, boolean written) { + this.recovered = recovered; + this.written = written; + } + } + + /** + * Appends to the open log, first closing off one a previous run left behind. + *

+ * Requires {@code ioLock}. + */ + Append append(byte[] bytes) { + assert Thread.holdsLock(ioLock); + // Recovered before the log is opened, because it may still be the one a previous run died writing. + // A commit on the caller's thread can get here before the processor's queued startup task has + // recovered it, and appended to, this run's events would sit behind that run's torn last frame. + int recovered = diskLog().isOpen() ? 0 : recoverInterruptedLog(); + try { + diskLog().append(bytes); + return new Append(recovered, true); + } catch (IOException | RuntimeException e) { + // Never let a full disk take the application down. The SDK gives up on persistence for the + // rest of the session instead, which is how other SDKs have crashed their hosts. + logger.warn("Giving up on persisting events: {}", LogValues.exceptionSummary(e)); + return new Append(recovered, false); + } + } + + /** What closing the open log produced. */ + static final class LogClose { + /** The batch the log became, or null where the close failed. */ + final EventStore.Batch batch; + /** + * Where the close failed, whether the log is still there to try again with. Where it is not - a + * directory the platform reclaimed, say - its events are gone, and they have to stop counting + * against capacity or the store would refuse events for the rest of the session. + */ + final boolean logSurvives; + + private LogClose(EventStore.Batch batch, boolean logSurvives) { + this.batch = batch; + this.logSurvives = logSurvives; + } + } + + /** + * Closes the open log into a batch, so the next append starts a new one. + *

+ * Requires {@code ioLock}. + * + * @param events how many events the log holds, which the caller has counted as it staged them + */ + LogClose closeLog(int events) { + assert Thread.holdsLock(ioLock); + String payloadId = UUID.randomUUID().toString(); + File file = newBatchFile(payloadId, events); + if (!diskLog().closeMovingTo(file)) { + logger.warn("Could not close the event log for delivery"); + return new LogClose(null, diskLog().exists()); + } + + stored.put(payloadId, new Stored(file, events)); + return new LogClose(new EventStore.Batch(payloadId, events), true); + } + + /** Closes the descriptor, leaving everything where it is. Requires {@code ioLock}. */ + void closeOpenLog() { + assert Thread.holdsLock(ioLock); + // Not diskLog(): a store that never wrote has never asked the platform where it lives, and shutting + // down is no reason to start. + if (diskLog != null) { + diskLog.closeOutput(); + } + } + + // MARK: listing and reading + + /** + * Requires {@code ioLock}. + * + * @return the batches the directory holds, oldest first, which is the order they are delivered in + */ + List listBatches() { + assert Thread.holdsLock(ioLock); + List batches = new ArrayList<>(); + for (File file : readyFilesOldestFirst()) { + int events = eventsInName(file.getName()); + String payloadId = payloadIdInName(file.getName(), events); + if (isDeliveredButUndeleted(payloadId, file)) { + continue; + } + // Reading is only for a batch whose name does not count it, which a version that named them + // without a count may have left behind, and then only the once. + if (events < 0) { + Stored known = stored.get(payloadId); + if (known != null) { + events = known.events; + } else { + byte[] log; + try { + log = readFile(file); + } catch (IOException e) { + // Left for a later listing rather than deleted: the read may pass, and the events in + // it may be intact. Without a count it cannot be delivered now. + logger.warn("Could not read stored events, will try again later: {}", + LogValues.exceptionSummary(e)); + continue; + } + if (log == null) { + // Delivered and removed by another process since the listing. + continue; + } + events = EventLogFormat.eventCount(log); + } + if (events < 0) { + // Written by a version whose format this one does not read, or damaged beyond what the + // torn-tail recovery tolerates. Either way it can never be delivered. + deleteQuietly(file); + continue; + } + } + stored.put(payloadId, new Stored(file, events)); + batches.add(new EventStore.Batch(payloadId, events)); + } + + // A batch that left the directory without going through remove() would otherwise keep its count + // for the rest of the session. + Set listed = new HashSet<>(); + for (EventStore.Batch batch : batches) { + listed.add(batch.payloadId); + } + stored.keySet().retainAll(listed); + return batches; + } + + /** + * Whether a batch has already been delivered and only its file is left, trying the delete that failed once + * more on the way. + *

+ * Requires {@code ioLock}. + */ + private boolean isDeliveredButUndeleted(String payloadId, File file) { + if (!deliveredButUndeleted.contains(payloadId)) { + return false; + } + deleteQuietly(file); + if (!file.exists()) { + deliveredButUndeleted.remove(payloadId); + } + return true; + } + + /** + * Requires {@code ioLock}. + * + * @return the JSON request body for a batch, or null if it is gone or holds no events + * @throws EventStore.UnreadableBatchException if the batch is there but holds nothing this version can + * send + * @throws IOException if the batch is there but could not be read this time + */ + byte[] body(String payloadId, int events) throws IOException { + assert Thread.holdsLock(ioLock); + byte[] log = readFile(batchFile(payloadId, events)); + if (log == null) { + return null; + } + byte[] body = EventLogFormat.assembleBody(log); + // A log this version reads that holds no events has nothing to lose. + if (body == null && EventLogFormat.eventCount(log) != 0) { + throw new EventStore.UnreadableBatchException(payloadId); + } + return body; + } + + /** + * Forgets a batch. Requires {@code ioLock}. + * + * @return the events it stopped counting, which is zero for a batch that was already gone - one another + * process delivered, say, which a listing has stopped counting already + */ + int remove(String payloadId, int events) { + assert Thread.holdsLock(ioLock); + File file = batchFile(payloadId, events); + deleteQuietly(file); + if (file.exists()) { + // Its events stop counting all the same: LaunchDarkly has them, and holding capacity for a file this + // process may never be allowed to delete would cost the session events it could still have recorded. + logger.debug("Could not delete a delivered batch; it will not be sent again this session"); + deliveredButUndeleted.add(payloadId); + } + Stored removed = stored.remove(payloadId); + return removed == null ? 0 : removed.events; + } + + /** + * Every event in the directory, encoded exactly as it will be sent. Requires {@code ioLock}. + *

+ * For tests and for diagnosing a store rather than for the recording path: it reads every file. + */ + List storedPayloads() { + assert Thread.holdsLock(ioLock); + List logs = new ArrayList<>(); + for (File file : readyFilesOldestFirst()) { + logs.add(readFileOrNull(file)); + } + if (diskLog().exists()) { + logs.add(readFileOrNull(diskLog().file())); + } + + List payloads = new ArrayList<>(); + for (byte[] log : logs) { + payloads.addAll(EventLogFormat.eventPayloads(log)); + } + return payloads; + } + + /** + * Requires {@code ioLock}. + * + * @return every batch file in the directory, oldest first + */ + private List readyFilesOldestFirst() { + List ready = new ArrayList<>(); + File[] files = directory().listFiles(); + if (files == null) { + return ready; + } + for (File file : files) { + if (file.getName().startsWith(BATCH_PREFIX)) { + ready.add(file); + } + } + Collections.sort(ready, new Comparator() { + @Override + public int compare(File a, File b) { + return Long.compare(a.lastModified(), b.lastModified()); + } + }); + return ready; + } + + /** + * @return the file's bytes, or null if it no longer exists + * @throws IOException if it exists but could not be read. That may pass, and the file is intact: + * a process out of file descriptors fails to open it with the same exception a missing file + * gets, which is why existence is checked rather than the exception's type. + */ + private byte[] readFile(File file) throws IOException { + ByteArrayOutputStream contents = new ByteArrayOutputStream(); + byte[] chunk = new byte[8192]; + try (InputStream in = reader.openForReading(file)) { + int read; + while ((read = in.read(chunk)) > 0) { + contents.write(chunk, 0, read); + } + } catch (IOException e) { + if (!file.exists()) { + return null; + } + throw e; + } + return contents.toByteArray(); + } + + /** For inspecting the store, where a file that cannot be read is simply left out. */ + private byte[] readFileOrNull(File file) { + try { + return readFile(file); + } catch (IOException e) { + return null; + } + } + + private void deleteQuietly(File file) { + if (file.exists() && !file.delete()) { + logger.debug("Could not delete {}", file); + } + } + + // MARK: recovery + + /** + * Closes off any log left open by a previous run of this process, so its events join the batches to + * deliver. + *

+ * The events in it were recorded by a process that is gone, so there is no one left to add to it. Only + * this process's own log is touched: another process's may still be open in a process that is alive. + *

+ * Requires {@code ioLock}. Does its work at most once, and always before this process first opens its + * log: after that, the log under this name holds this run's events, and closing it as a previous run's + * would send them early and report them as recovered. + * + * @return how many events it closed off, which is zero on every call after the first + */ + int recoverInterruptedLog() { + assert Thread.holdsLock(ioLock); + if (recoveryDone) { + return 0; + } + recoveryDone = true; + + File log = diskLog().file(); + if (!log.exists()) { + return 0; + } + int events; + try { + events = EventLogFormat.eventCount(readFile(log)); + } catch (IOException e) { + // It cannot stay under this name, which this process is about to append its own events to, + // and deleting it would lose events that may be intact. Closed uncounted instead, it is read + // and counted by whichever listing first manages to. + logger.warn("Could not read events from a previous run, will try again later: {}", + LogValues.exceptionSummary(e)); + // Named without a count, which is what a batch whose events have never been counted is: the + // listing that first manages to read it is what counts it. + log.renameTo(new File(directory(), BATCH_PREFIX + UUID.randomUUID())); + return 0; + } + if (events < 0) { + // Unreadable, and a log that cannot be read cannot be appended to either. + deleteQuietly(log); + return 0; + } + if (events == 0) { + deleteQuietly(log); + return 0; + } + String payloadId = UUID.randomUUID().toString(); + File file = newBatchFile(payloadId, events); + if (!log.renameTo(file)) { + return 0; + } + stored.put(payloadId, new Stored(file, events)); + logger.info("Recovered {} event(s) that a previous run of this application did not deliver", + events); + return events; + } + + // MARK: naming + + /** The directory the store for one environment belongs in. */ + static File defaultDirectory(File noBackupFilesDir, String mobileKey) { + return new File(new File(noBackupFilesDir, DIRECTORY_NAME), environmentDirectoryName(mobileKey)); + } + + /** + * Names the directory for one environment, hashed so the mobile key itself is never a filename. + *

+ * Deliberately not {@link LDUtil#urlSafeBase64Hash(String)}, which goes through the Android + * framework's Base64 and so is unavailable to plain JVM tests, and which pads with a character + * that has no business in a path. + */ + private static String environmentDirectoryName(String mobileKey) { + // Half a digest, which is far more than enough to keep two mobile keys apart. + return hexDigest(mobileKey, 16); + } + + /** + * @param bytes how many bytes of the digest to render, so a caller can trade length for headroom + * @return the first {@code bytes} bytes of the input's SHA-256, in lowercase hex + */ + static String hexDigest(String input, int bytes) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256") + .digest(input.getBytes(StandardCharsets.UTF_8)); + StringBuilder hex = new StringBuilder(bytes * 2); + for (int i = 0; i < bytes; i++) { + hex.append(Character.forDigit((hash[i] >> 4) & 0xf, 16)); + hex.append(Character.forDigit(hash[i] & 0xf, 16)); + } + return hex.toString(); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); // shouldn't be possible; SHA-256 is built in + } + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventLogFormat.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventLogFormat.java new file mode 100644 index 000000000..aeb2a5d96 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventLogFormat.java @@ -0,0 +1,214 @@ +package com.launchdarkly.sdk.android; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * The on-disk shape of an event log. + *

+ * A log opens with a magic and a format version, and holds a sequence of frames: + * 

+ *     +- 4 bytes -+- 2 bytes -+     +- 2 bytes -+-  4 bytes  -+- n bytes -+
+ *     |   LDEV    |  version  | ... |   type    |  length (n) |  payload  |
+ *     +-----------+-----------+     +-----------+-------------+-----------+
+ *
+ * The length prefix is what makes a log recoverable: a reader can tell a frame the writer finished + * from one it did not, without having to parse the event to find where it ends. The version is what + * makes an upgrade safe - a log this version does not understand is discarded rather than misread - + * and the frame type leaves room for a later version to write something new into a log this one + * still reads. + *

+ * The format is byte-for-byte the one the iOS SDK writes, so the two can be reasoned about, and + * debugged, as one thing. + */ +final class EventLogFormat { + /** Bump this whenever the framing or the meaning of a frame changes. */ + static final int VERSION = 1; + static final byte[] MAGIC = "LDEV".getBytes(StandardCharsets.US_ASCII); + static final int FILE_HEADER_SIZE = 6; + static final int FRAME_HEADER_SIZE = 6; + /** The only frame type written today; a reader skips a frame whose type it does not know. */ + static final int EVENT_FRAME = 1; + /** A ceiling on a single frame, so a corrupt length cannot make recovery allocate wildly. */ + static final int MAX_FRAME_SIZE = 8 * 1024 * 1024; + + private EventLogFormat() { + } + + static byte[] fileHeader() { + byte[] header = new byte[FILE_HEADER_SIZE]; + System.arraycopy(MAGIC, 0, header, 0, MAGIC.length); + header[4] = (byte) (VERSION >> 8); + header[5] = (byte) VERSION; + return header; + } + + static void writeFrame(ByteArrayOutputStream target, byte[] serializedEvent) { + target.write((byte) (EVENT_FRAME >> 8)); + target.write((byte) EVENT_FRAME); + int length = serializedEvent.length; + target.write((byte) (length >> 24)); + target.write((byte) (length >> 16)); + target.write((byte) (length >> 8)); + target.write((byte) length); + target.write(serializedEvent, 0, length); + } + + /** Frames on their own, as a log: only for the in-memory fallback, which has no file. */ + static byte[] withFileHeader(byte[] frames) { + byte[] log = new byte[FILE_HEADER_SIZE + frames.length]; + System.arraycopy(fileHeader(), 0, log, 0, FILE_HEADER_SIZE); + System.arraycopy(frames, 0, log, FILE_HEADER_SIZE, frames.length); + return log; + } + + /** + * @return how many event frames the log holds, or -1 if it is not a log this version reads + */ + static int eventCount(byte[] log) { + Frames frames = new Frames(log); + if (!frames.isReadable()) { + return -1; + } + int events = 0; + while (frames.next()) { + if (frames.type() == EVENT_FRAME) { + events++; + } + } + return events; + } + + /** + * Assembles the JSON array LaunchDarkly expects out of the frames, without parsing the events: + * they were serialized on the way in and are shipped exactly as they were recorded. + * + * @return the request body, or null if the log is unreadable or holds no events + */ + static byte[] assembleBody(byte[] log) { + Frames frames = new Frames(log); + if (!frames.isReadable()) { + return null; + } + ByteArrayOutputStream body = new ByteArrayOutputStream(log.length + 2); + body.write('['); + boolean isFirst = true; + while (frames.next()) { + if (frames.type() != EVENT_FRAME) { + continue; + } + if (!isFirst) { + body.write(','); + } + body.write(log, frames.start(), frames.length()); + isFirst = false; + } + if (isFirst) { + return null; + } + body.write(']'); + return body.toByteArray(); + } + + /** + * @return each event in the log, serialized as it will be sent; empty if the log is unreadable + */ + static List eventPayloads(byte[] log) { + List payloads = new ArrayList<>(); + Frames frames = new Frames(log); + if (!frames.isReadable()) { + return payloads; + } + while (frames.next()) { + if (frames.type() == EVENT_FRAME) { + payloads.add(Arrays.copyOfRange(log, frames.start(), frames.start() + frames.length())); + } + } + return payloads; + } + + /** + * Walks the frames of a log, stopping at the first one that is not wholly there. + *

+ * A process that died partway through a write leaves a torn frame at the end. Everything before it + * is intact, so recovery keeps that and drops only the tail. + *

+ * A cursor rather than a callback, so that a caller counting or copying frames does it in a plain + * loop. {@link #next()} is false from the first frame that is not wholly there, and always for a log + * that is not {@link #isReadable() readable}. + */ + static final class Frames { + private final byte[] log; + private final boolean readable; + private int cursor = FILE_HEADER_SIZE; + private int type; + private int start; + private int length; + + Frames(byte[] log) { + this.log = log; + this.readable = isReadable(log); + } + + /** Whether this is a log this version reads at all. */ + boolean isReadable() { + return readable; + } + + /** Moves to the next whole frame, if there is one. */ + boolean next() { + if (!readable || cursor + FRAME_HEADER_SIZE > log.length) { + return false; + } + int frameType = readUInt16(log, cursor); + int frameLength = readInt32(log, cursor + 2); + int frameStart = cursor + FRAME_HEADER_SIZE; + if (frameLength <= 0 || frameLength > MAX_FRAME_SIZE || frameStart + frameLength > log.length) { + cursor = log.length; + return false; + } + type = frameType; + start = frameStart; + length = frameLength; + cursor = frameStart + frameLength; + return true; + } + + int type() { + return type; + } + + /** Where the current frame's payload starts in the log. */ + int start() { + return start; + } + + int length() { + return length; + } + + private static boolean isReadable(byte[] log) { + if (log == null || log.length < FILE_HEADER_SIZE) { + return false; + } + for (int i = 0; i < MAGIC.length; i++) { + if (log[i] != MAGIC[i]) { + return false; + } + } + return readUInt16(log, MAGIC.length) == VERSION; + } + } + + private static int readUInt16(byte[] bytes, int offset) { + return ((bytes[offset] & 0xff) << 8) | (bytes[offset + 1] & 0xff); + } + + private static int readInt32(byte[] bytes, int offset) { + return ((bytes[offset] & 0xff) << 24) | ((bytes[offset + 1] & 0xff) << 16) + | ((bytes[offset + 2] & 0xff) << 8) | (bytes[offset + 3] & 0xff); + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventStore.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventStore.java new file mode 100644 index 000000000..f3423d492 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventStore.java @@ -0,0 +1,799 @@ +package com.launchdarkly.sdk.android; + +import androidx.annotation.VisibleForTesting; + +import com.launchdarkly.logging.LDLogger; + +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +/** + * A crash-durable append log of serialized analytics events. + *

+ * Events are appended to an open log, which is closed into a batch when the event processor is ready to + * deliver one. A batch is deleted only once LaunchDarkly has accepted it, so a delivery interrupted by + * the process dying is retried on the next run rather than lost. + *

+ * Bytes reach the kernel through an ordinary write and are never {@code fsync}ed. What this defends + * against is the process dying - which is what an uncaught exception, an ANR kill, a native crash, and + * the system reclaiming a backgrounded application all are - and a process cannot take back bytes the + * kernel already holds. Only losing the kernel itself, to a panic or a power cut, can discard them, and + * covering that would mean an {@code fsync} per event, which costs milliseconds where this costs + * microseconds. + *

+ * Several clients. An application configured for more than one environment gets one + * {@code LDClient} and one store per mobile key, and the mobile key's hash is part of the directory, so + * two environments never read or write each other's events. + *

+ * Several processes. An application may declare components in more than one process, and each + * gets its own copy of the SDK that shares nothing but the filesystem. Only the open log is per-process, + * named after the process appending to it, which is what keeps two processes from interleaving frames in + * one file and keeps one process's startup recovery from stealing a log another is still writing to. + * Closed batches are deliberately left shared: a process that recorded events and then died may never + * run again, and a batch nobody but that process could see would never be delivered. If two processes + * do deliver the same batch, its payload ID is the same both times, and LaunchDarkly discards a repeated + * payload ID rather than counting the events twice. + *

+ * What a log looks like on disk is {@link EventLogFormat}; the log this process appends to is + * {@link DiskEventLog}; the directory the two of them live in is {@link EventDirectory}. This class is + * what decides when events move between memory, that directory, and the batches closed off for delivery, + * and what counts them against capacity on the way. It has no directory at all where the application asked + * for events to be kept off the disk, which is what keeps the filesystem out of that case entirely. + */ +class EventStore implements Closeable { + /** + * Where the events are kept, or null where the application asked for them to be kept off the disk. + *

+ * Everything about the filesystem is behind this, so that the store itself is the buffer, the capacity + * arithmetic, and the batches it holds in memory. A store with no disk runs the way the SDK ran before + * persistence existed, and never touches the filesystem, not even to look for a previous run's events. + */ + private final EventDirectory disk; + private final int capacity; + private final LDLogger logger; + + /** + * Guards the buffer and the counters. Held for a copy and never across a write, so an + * evaluation recording an event waits on another thread's array copy at worst, never on the disk. + */ + private final Object bufferLock = new Object(); + /** + * Serializes writes so that two threads committing at once cannot interleave a frame. Taken before + * {@link #bufferLock}, never after it. + */ + private final Object ioLock = new Object(); + + // Guarded by bufferLock. + private final ByteArrayOutputStream bufferData = new ByteArrayOutputStream(); + private int bufferedEventCount; + /** + * Events taken out of the buffer to be written to the log, or assembled into a batch held in + * memory, and not yet counted where they land. Both happen outside this lock, and for as long as + * they take the events are in neither place; without this they would be counted nowhere, and + * capacity would admit that many more. + */ + private int inTransitEvents; + private int committedEvents; + /** + * Events in closed batches: those the {@link #disk} is holding plus those in {@link #inMemoryBatches}. + * Changed only in the critical section that changes one of those, so that it cannot drift from them; a + * total worked out from them and written afterwards would overwrite a close or a removal in between. + */ + private int closedEvents; + /** + * Closed batches whose request is out, by payload ID, and the events in them. Every one is also in + * {@link #closedEvents}, and leaves both in the same critical section. + */ + private final Map sendingBatches = new HashMap<>(); + private int sendingEvents; + /** + * Events a caller has taken to encode and promised to stage, counted against capacity until each is + * staged. Staging one moves it from here to {@link #bufferedEventCount} under one lock, so an event + * on its way in is counted exactly once throughout. + */ + private int reservedEvents; + /** + * Whether events are being written to disk, which is what the application asked for until a write + * fails and the store gives up on persistence for the rest of the session. + *

+ * Written under {@link #bufferLock} but volatile, so {@link #isPersisting} can answer without it: + * every {@code track} asks, and a commit takes that lock once for each event it stages. + */ + private volatile boolean persistEvents; + + /** + * Guards {@link #inMemoryBatches}. + *

+ * Its own lock rather than {@code ioLock}, so that a store the application told to keep off the disk - + * where held batches are the only ones there are - never waits on the lock disk I/O is serialized + * under. Taken inside {@code ioLock} where there is a disk, never the other way round, and taken + * before {@code bufferLock}, never after it. + */ + private final Object heldLock = new Object(); + /** + * Batches held in memory because the filesystem would not take them, or because it was never asked. + *

+ * Persistence failing should cost durability and nothing else, so the store falls back to what the + * SDK did before it existed: hold the events, deliver them, lose them only if the process dies. + *

+ * Only to be used while holding {@link #heldLock}. + */ + private final Map inMemoryBatches = new LinkedHashMap<>(); + + EventStore(final File directory, final String processName, int capacity, boolean persistEvents, + LDLogger logger) { + this(new EventDirectory.Location() { + @Override + public File directory() { + return directory; + } + + @Override + public String processName() { + return processName; + } + }, capacity, persistEvents, logger); + } + + private EventStore(EventDirectory.Location location, int capacity, boolean persistEvents, + LDLogger logger) { + this.capacity = capacity >= 0 ? capacity : 1; + this.logger = logger; + // An application that has not asked for persistence gets the same store running the same way it + // runs once a write has failed: events are held, delivered, and lost only if the process dies. + // It also skips every read, so batches a run with persistence turned on left behind stay on disk + // until persistence is turned back on. + this.persistEvents = persistEvents; + this.disk = persistEvents + ? new EventDirectory(location, ioLock, logger, this::openForReading, this::openForAppending) + : null; + } + + /** + * Creates the store for one environment of one process. + * + * @param noBackupFilesDir where the platform lets the SDK keep files it must not lose + * @param mobileKey identifies the environment, so several clients stay out of each other's way + * @param processName identifies the process, so several processes stay out of each other's way + * @param persistEvents whether the application asked for events to outlive the process + */ + static EventStore create( + File noBackupFilesDir, + String mobileKey, + String processName, + int capacity, + boolean persistEvents, + LDLogger logger + ) { + return new EventStore(EventDirectory.defaultDirectory(noBackupFilesDir, mobileKey), processName, + capacity, persistEvents, logger); + } + + /** + * Creates the store for one environment of one process, leaving the platform to be asked where on + * first use, from one of the store's own threads. + * + * @param platformState where the no-backup directory and the process name come from + * @param mobileKey identifies the environment, so several clients stay out of each other's way + * @param persistEvents whether the application asked for events to outlive the process + */ + static EventStore create( + final PlatformState platformState, + String mobileKey, + int capacity, + boolean persistEvents, + LDLogger logger + ) { + EventDirectory.Location location = new EventDirectory.Location() { + @Override + public File directory() { + return EventDirectory.defaultDirectory(platformState.getNoBackupFilesDir(), mobileKey); + } + + @Override + public String processName() { + return platformState.getProcessName(); + } + }; + return new EventStore(location, capacity, persistEvents, logger); + } + + // MARK: recording + + /** + * A batch being sent is left out so that a flush makes room as soon as it sends, as it did when events + * were only held in memory; otherwise an event recorded while the request is out is refused for a batch + * LaunchDarkly may already have. A send that fails puts the batch back, so a full store can briefly + * hold capacity plus that batch. + * + * @return serialized events counted against capacity: those recorded and not yet accepted by + * LaunchDarkly, whether staged in memory, on their way out of it, written to the open log, or sitting + * in a closed batch - except a batch whose request is out + */ + int getPendingEventCount() { + synchronized (bufferLock) { + return countedEventsHoldingBufferLock(); + } + } + + /** + * Every event counted against capacity. The one place the counters are added up, so that neither + * reader can miss one of them. Requires {@link #bufferLock}. + */ + private int countedEventsHoldingBufferLock() { + // Clamped because a batch another process delivered can be marked as sending after a listing + // stopped counting it. + return bufferedEventCount + inTransitEvents + committedEvents + + Math.max(0, closedEvents - sendingEvents) + reservedEvents; + } + + /** + * Counts events against capacity before they are staged, for a caller that has taken them from + * somewhere capacity was already counting them and will encode them before staging. + *

+ * Each {@link #stageReserved} uses one up, and {@link #releaseReservations} gives back any that + * never made it, such as an event that could not be serialized. + */ + void reserve(int events) { + synchronized (bufferLock) { + reservedEvents += events; + } + } + + /** Gives back every reservation not used up by {@link #stageReserved}. */ + void releaseReservations() { + synchronized (bufferLock) { + reservedEvents = 0; + } + } + + /** + * @return whether a commit writes anything, which is false where persistence is off and once a write + * has failed and the store has given up on it + */ + boolean isPersisting() { + return persistEvents; + } + + /** + * Stages an already serialized event. + *

+ * Staging is a copy into a buffer and never writes, however much accumulates; {@link #commit()} is what + * makes the event outlive the process, so how many bytes one write carries is the caller's decision. + * + * @return false if the event was dropped because capacity is reached + */ + boolean stage(byte[] serializedEvent) { + return stage(serializedEvent, false); + } + + /** + * @param bypassingCapacity true for an event that must be recorded even when the store is full + */ + boolean stage(byte[] serializedEvent, boolean bypassingCapacity) { + return stage(serializedEvent, bypassingCapacity, false); + } + + /** + * Stages an event {@link #reserve} already counted, turning its reservation into a staged event in + * one step. Bypasses capacity, which the reservation already answered to. + */ + boolean stageReserved(byte[] serializedEvent) { + return stage(serializedEvent, true, true); + } + + private boolean stage(byte[] serializedEvent, boolean bypassingCapacity, boolean reserved) { + if (serializedEvent == null || serializedEvent.length == 0 + || serializedEvent.length > EventLogFormat.MAX_FRAME_SIZE) { + return false; + } + + synchronized (bufferLock) { + if (!bypassingCapacity && countedEventsHoldingBufferLock() >= capacity) { + return false; + } + EventLogFormat.writeFrame(bufferData, serializedEvent); + bufferedEventCount++; + if (reserved && reservedEvents > 0) { + reservedEvents--; + } + } + return true; + } + + /** + * Hands every staged byte to the kernel, so the events recorded so far survive the process dying. + */ + void commit() { + // Answered before ioLock, which a delivery can hold for as long as it takes to read a batch + // back: with persistence off there is nothing to write, so nothing worth waiting for. + synchronized (bufferLock) { + if (!persistEvents) { + return; + } + } + synchronized (ioLock) { + commitHoldingIoLock(); + } + } + + // MARK: delivery + + /** + * Commits, then closes the open log into a batch to deliver, and whatever is still staged in memory + * into another. + *

+ * Both can hold events at once. A write that fails after earlier ones succeeded gives up on + * persistence with those earlier events in the log and the failed ones back in memory, and closing + * only the log would leave the rest for a delivery that reports success without them. + * + * @return every batch this closed, oldest first: the one from the log, then the one from memory, so + * at most two and usually one; empty when there is nothing to send + */ + List closeBatch() { + List closed = new ArrayList<>(1); + // Where the application asked for no disk there is no log to close, and so nothing to close it + // alongside. + if (disk == null) { + addIfClosed(closed, closeInMemoryBatch()); + return closed; + } + synchronized (ioLock) { + commitHoldingIoLock(); + + int events; + int stillBuffered; + synchronized (bufferLock) { + events = committedEvents; + stillBuffered = bufferedEventCount; + } + + if (events > 0) { + addIfClosed(closed, closeOpenLogHoldingIoLock(events)); + } + // Only reachable once persistence has been given up on; a healthy commit leaves nothing + // staged behind. Closed after the log, so the older events are delivered first. + if (stillBuffered > 0) { + addIfClosed(closed, closeInMemoryBatch()); + } + return closed; + } + } + + private static void addIfClosed(List closed, Batch batch) { + if (batch != null) { + closed.add(batch); + } + } + + /** Requires {@code ioLock}. */ + private Batch closeOpenLogHoldingIoLock(int events) { + EventDirectory.LogClose closed = disk.closeLog(events); + if (closed.batch == null) { + if (!closed.logSurvives) { + // The log itself is gone, so its events are too. They have to stop counting against + // capacity or the store would refuse events for the rest of the session. + synchronized (bufferLock) { + committedEvents = 0; + } + } + return null; + } + + synchronized (bufferLock) { + committedEvents = 0; + closedEvents += events; + } + return closed.batch; + } + + /** + * Closes whatever is staged into a batch that never reaches a file. + *

+ * Called under {@code ioLock} where there is a disk, so that it and the log are closed as one step, + * and on its own where there is not. + */ + private Batch closeInMemoryBatch() { + byte[] frames; + int events; + synchronized (bufferLock) { + frames = bufferData.toByteArray(); + events = bufferedEventCount; + bufferData.reset(); + bufferedEventCount = 0; + inTransitEvents += events; + } + String payloadId = UUID.randomUUID().toString(); + boolean held = false; + try { + // Outside bufferLock, because it copies every staged frame; inTransitEvents is what keeps + // them counted meanwhile. + byte[] body = EventLogFormat.assembleBody(EventLogFormat.withFileHeader(frames)); + if (body == null) { + return null; + } + synchronized (heldLock) { + inMemoryBatches.put(payloadId, new HeldBatch(body, events)); + synchronized (bufferLock) { + inTransitEvents -= events; + closedEvents += events; + } + } + held = true; + } finally { + if (!held) { + synchronized (bufferLock) { + inTransitEvents -= events; + } + } + } + return new Batch(payloadId, events); + } + + /** A batch the filesystem would not take, kept where the SDK used to keep all of them. */ + private static final class HeldBatch { + final byte[] body; + final int eventCount; + + HeldBatch(byte[] body, int eventCount) { + this.body = body; + this.eventCount = eventCount; + } + } + + /** + * @return batches awaiting delivery, oldest first, including any left by a previous run + */ + List pendingBatches() { + // Listed in one critical section where there is a disk, so that the files and what is held cannot + // be counted either side of a close that moves events between them. + List batches; + if (disk != null) { + synchronized (ioLock) { + batches = disk.listBatches(); + addHeldBatchesAndRecount(batches); + } + } else { + batches = new ArrayList<>(); + addHeldBatchesAndRecount(batches); + } + return batches; + } + + /** + * Adds the batches the filesystem would not take, after the files: anything held was closed by this run + * and so is newer than whatever a previous run left on the disk. Then counts everything listed as + * closed, which is how batches another process closed or delivered come to be counted or not. + *

+ * Requires {@code ioLock} where there is a disk, so that the count is of the listing as it stands. + */ + private void addHeldBatchesAndRecount(List batches) { + synchronized (heldLock) { + for (Map.Entry held : inMemoryBatches.entrySet()) { + batches.add(new Batch(held.getKey(), held.getValue().eventCount)); + } + int total = 0; + Set listed = new HashSet<>(); + for (Batch batch : batches) { + total += batch.eventCount; + listed.add(batch.payloadId); + } + synchronized (bufferLock) { + closedEvents = total; + // One no longer listed is no longer counted, so it cannot be left out of the count either. + for (String payloadId : new ArrayList<>(sendingBatches.keySet())) { + if (!listed.contains(payloadId)) { + forgetSendingHoldingBufferLock(payloadId); + } + } + } + } + } + + /** + * A batch that is there to read and holds nothing this version can send: damaged, or written in a + * format it does not read. Unlike a batch that is gone, its events are lost, and unlike any other + * {@link IOException}, no later attempt changes that. + */ + static final class UnreadableBatchException extends IOException { + UnreadableBatchException(String payloadId) { + super("batch " + payloadId + " holds nothing this version can send"); + } + } + + /** + * @return the JSON request body for a batch, or null if it is gone or holds no events, neither of which + * loses anything + * @throws UnreadableBatchException if the batch is there but holds nothing this version can send + * @throws IOException if the batch is there but could not be read this time, which a later attempt may + * manage + */ + byte[] body(Batch batch) throws IOException { + synchronized (heldLock) { + HeldBatch held = inMemoryBatches.get(batch.payloadId); + if (held != null) { + return held.body; + } + } + + // A batch is either held or on the disk, never both and never moved between them, so a batch that + // is not held in a store with no disk does not exist. + if (disk == null) { + return null; + } + + synchronized (ioLock) { + return disk.body(batch.payloadId, batch.eventCount); + } + } + + /** + * Forgets a batch, which is only correct once LaunchDarkly has accepted it or permanently refused it. + */ + void remove(Batch batch) { + // Only a count this call takes away stops counting. Two deliveries can both get to the same batch, + // and a listing drops a batch another process delivered before this one gets to remove it. + synchronized (heldLock) { + HeldBatch held = inMemoryBatches.remove(batch.payloadId); + if (held != null) { + synchronized (bufferLock) { + closedEvents -= held.eventCount; + forgetSendingHoldingBufferLock(batch.payloadId); + } + return; + } + } + + if (disk == null) { + synchronized (bufferLock) { + forgetSendingHoldingBufferLock(batch.payloadId); + } + return; + } + synchronized (ioLock) { + int counted = disk.remove(batch.payloadId, batch.eventCount); + synchronized (bufferLock) { + closedEvents -= counted; + forgetSendingHoldingBufferLock(batch.payloadId); + } + } + } + + /** + * Stops counting a batch against capacity while its request is out. Ended by {@link #remove} or + * {@link #stoppedSending}. + */ + void startedSending(Batch batch) { + synchronized (bufferLock) { + if (!sendingBatches.containsKey(batch.payloadId)) { + sendingBatches.put(batch.payloadId, batch.eventCount); + sendingEvents += batch.eventCount; + } + } + } + + /** + * Counts a batch against capacity again, after a send that did not end with it removed. Does nothing + * for a batch {@link #remove} already forgot. + */ + void stoppedSending(Batch batch) { + synchronized (bufferLock) { + forgetSendingHoldingBufferLock(batch.payloadId); + } + } + + /** + * Requires {@link #bufferLock}. + */ + private void forgetSendingHoldingBufferLock(String payloadId) { + Integer events = sendingBatches.remove(payloadId); + if (events != null) { + sendingEvents -= events; + } + } + + /** + * Closes any log this process left open on a previous run, so its events join the batches to deliver. + *

+ * The events in it were recorded by a process that is gone, so there is no one left to add to it. Only + * this process's own log is touched: another process's may still be open in a process that is alive. + */ + void recoverInterruptedLog() { + if (disk == null) { + return; + } + synchronized (ioLock) { + countRecoveredHoldingIoLock(disk.recoverInterruptedLog()); + } + } + + /** + * Counts events a recovery closed off into batches of their own, which belong to the closed batches + * from the moment the recovery moved them there. + *

+ * Requires {@code ioLock}. + */ + private void countRecoveredHoldingIoLock(int events) { + if (events <= 0) { + return; + } + synchronized (bufferLock) { + closedEvents += events; + } + } + + // MARK: writing + + /** Requires {@code ioLock}. */ + private void commitHoldingIoLock() { + byte[] bytes; + int events; + synchronized (bufferLock) { + if (!persistEvents || bufferedEventCount == 0) { + // Staged bytes are left where they are: with nowhere durable to put them, memory is + // better than dropping them. + return; + } + bytes = bufferData.toByteArray(); + events = bufferedEventCount; + bufferData.reset(); + bufferedEventCount = 0; + inTransitEvents += events; + } + + boolean written = false; + try { + written = append(bytes); + } finally { + // Counted where they landed in the same critical section that stops counting them in + // transit, so a capacity check never sees them in both places or in neither. + synchronized (bufferLock) { + inTransitEvents -= events; + if (written) { + committedEvents += events; + } else { + // Put back, to be delivered from memory rather than lost. + byte[] laterFrames = bufferData.toByteArray(); + bufferData.reset(); + writeQuietly(bufferData, bytes); + writeQuietly(bufferData, laterFrames); + bufferedEventCount += events; + } + } + } + } + + /** + * Requires {@code ioLock}. + * + * @return false if the bytes could not be written, in which case persistence is given up on + */ + private boolean append(byte[] bytes) { + synchronized (bufferLock) { + if (!persistEvents) { + return false; + } + } + EventDirectory.Append appended = disk.append(bytes); + countRecoveredHoldingIoLock(appended.recovered); + if (!appended.written) { + synchronized (bufferLock) { + persistEvents = false; + } + return false; + } + return true; + } + + /** Overridable so a test can make a write fail partway through a session, as a full disk does. */ + @VisibleForTesting + FileOutputStream openForAppending(File log) throws IOException { + return new FileOutputStream(log, true); + } + + @Override + public void close() { + synchronized (ioLock) { + commitHoldingIoLock(); + if (disk != null) { + disk.closeOpenLog(); + } + } + } + + /** @return where the events are kept, or null where the application asked for no disk */ + File getDirectory() { + if (disk == null) { + return null; + } + synchronized (ioLock) { + return disk.directory(); + } + } + + private static void writeQuietly(ByteArrayOutputStream target, byte[] bytes) { + try { + target.write(bytes); + } catch (IOException e) { + // ByteArrayOutputStream.write does not throw; this only satisfies the compiler. + } + } + + /** Overridable so a test can make a read fail while the file is intact. */ + @VisibleForTesting + InputStream openForReading(File file) throws IOException { + return new FileInputStream(file); + } + + /** + * A group of serialized events closed off for delivery. + *

+ * The identifier doubles as the payload ID sent to LaunchDarkly, so a batch retried after the process + * died mid-delivery is recognized upstream as the same delivery rather than counted twice. + */ + static final class Batch { + final String payloadId; + final int eventCount; + + Batch(String payloadId, int eventCount) { + this.payloadId = payloadId; + this.eventCount = eventCount; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Batch)) { + return false; + } + Batch o = (Batch) other; + return payloadId.equals(o.payloadId) && eventCount == o.eventCount; + } + + @Override + public int hashCode() { + return payloadId.hashCode() * 31 + eventCount; + } + + @Override + public String toString() { + return "Batch(" + payloadId + ", " + eventCount + " events)"; + } + } + + /** + * Every event the store is holding, serialized exactly as it will be sent. + *

+ * Reading the log means committing what is staged, so this is not free and is meant for tests and for + * diagnosing a store rather than for the recording path. + */ + List pendingEventPayloads() { + commit(); + + List payloads = new ArrayList<>(); + if (disk != null) { + synchronized (ioLock) { + payloads.addAll(disk.storedPayloads()); + } + } + synchronized (bufferLock) { + if (bufferedEventCount > 0) { + payloads.addAll(EventLogFormat.eventPayloads( + EventLogFormat.withFileHeader(bufferData.toByteArray()))); + } + } + return payloads; + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventUtil.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventUtil.java index e605de085..e4c71f6a3 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventUtil.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/EventUtil.java @@ -15,6 +15,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -22,17 +23,46 @@ abstract class EventUtil { static ScheduledExecutorService makeEventsTaskExecutor() { - return Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { + return Executors.newSingleThreadScheduledExecutor( + makeThreadFactory("LaunchDarkly-DefaultEventProcessor-%d")); + } + + /** + * A thread of its own for posting diagnostic events, kept apart from the one that delivers + * analytics events. + *

+ * A diagnostic post is an HTTP request like any other and can hold its thread for tens of seconds + * against a network that accepts connections and never answers. Sharing a thread with analytics + * delivery would let that stall the flushes, and an event buffer that is not being drained fills + * up and starts dropping what the application asked to send. This mirrors the separation + * {@code DefaultEventProcessor} has, where analytics go out on dedicated delivery workers and + * diagnostics on the shared executor. + */ + static ExecutorService makeDiagnosticsTaskExecutor() { + return Executors.newSingleThreadExecutor( + makeThreadFactory("LaunchDarkly-DiagnosticEventPoster-%d")); + } + + /** + * A thread of its own for the event commits no caller waits on, kept apart from the one that + * delivers, so that a write never queues behind a request held open by a slow network. + */ + static ExecutorService makeEventsCommitExecutor() { + return Executors.newSingleThreadExecutor( + makeThreadFactory("LaunchDarkly-event-store-%d")); + } + + private static ThreadFactory makeThreadFactory(String nameFormat) { + return new ThreadFactory() { final AtomicLong count = new AtomicLong(0); @Override public Thread newThread(@NonNull Runnable r) { Thread thread = Executors.defaultThreadFactory().newThread(r); - thread.setName(String.format(Locale.ROOT, "LaunchDarkly-DefaultEventProcessor-%d", - count.getAndIncrement())); + thread.setName(String.format(Locale.ROOT, nameFormat, count.getAndIncrement())); thread.setDaemon(true); return thread; } - }); + }; } static DiagnosticStore.SdkDiagnosticParams makeDiagnosticParams(ClientContext clientContext) { diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClient.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClient.java index f224668a1..18e1cda20 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClient.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClient.java @@ -6,6 +6,7 @@ import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; +import com.launchdarkly.logging.LDLogLevel; import com.launchdarkly.logging.LDLogger; import com.launchdarkly.logging.LogValues; import com.launchdarkly.sdk.EvaluationDetail; @@ -37,6 +38,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -84,6 +87,13 @@ public class LDClient implements LDClientInterface, Closeable { // If 15 seconds or more is passed as a timeout to init, we will log a warning. private static final int EXCESSIVE_INIT_WAIT_SECONDS = 15; + // Flags already reported unknown at info level. An application can poll a flag that does not exist + // on every redraw, and each of those would otherwise write to logcat. Bounded because the keys come + // from the application; past the bound an unknown flag is reported at debug only. + private static final int MAX_UNKNOWN_FLAGS_REPORTED = 100; + private final Set unknownFlagsReported = + Collections.newSetFromMap(new ConcurrentHashMap()); + // Shared by both registration paths, so that a plugin failing during init and the same plugin // failing under registerPlugin are reported identically. private static final String PLUGIN_GET_HOOKS_ERROR = @@ -641,7 +651,11 @@ private EvaluationDetail variationDetailInternal(@NonNull String key, @ EvaluationDetail result; if (flag == null) { - logger.info("Unknown feature flag \"{}\"; returning default value", key); + if (unknownFlagsReported.size() < MAX_UNKNOWN_FLAGS_REPORTED && unknownFlagsReported.add(key)) { + logger.info("Unknown feature flag \"{}\"; returning default value", key); + } else { + logger.debug("Unknown feature flag \"{}\"; returning default value", key); + } eventProcessor.recordEvaluationEvent(context, key, EventProcessor.NO_VERSION, EvaluationDetail.NO_VARIATION, defaultValue, null, defaultValue, false, null); @@ -700,7 +714,11 @@ private EvaluationDetail variationDetailInternal(@NonNull String key, @ ); } - logger.debug("returning variation: {} flagKey: {} context key: {}", result, key, context.getKey()); + // Checked first: three arguments go through the varargs overload, which builds its array whether + // or not debug logging is on. + if (logger.isEnabled(LDLogLevel.DEBUG)) { + logger.debug("returning variation: {} flagKey: {} context key: {}", result, key, context.getKey()); + } return result; } @@ -779,6 +797,55 @@ private void flushInternal() { eventProcessor.flush(); } + @Override + public boolean flushAndWait(long timeout, TimeUnit unit) { + // Clamped because toNanos saturates: a timeout at Long.MIN_VALUE nanos would make the + // remaining time below underflow, and wrap round to a wait with no bound at all. + long deadline = System.nanoTime() + Math.max(0, unit.toNanos(timeout)); + Map clients = getInstancesIfTheyIncludeThisClient(); + if (clients.isEmpty()) { + // This client has been closed, or replaced by a later init; either way it can deliver + // nothing, and saying otherwise would tell the caller its events were safe. + return false; + } + // Every environment is started before any of them is waited on. Each has its own event + // processor and its own thread, so waiting on one before starting the next would spend the + // caller's budget on deliveries that could have been running all along. + List> deliveries = new ArrayList<>(clients.size()); + for (LDClient client : clients.values()) { + deliveries.add(client.eventProcessor.flushAsync()); + } + boolean delivered = true; + for (Future delivery : deliveries) { + // Each wait gets what is left of the one budget rather than a fresh copy of it, so that + // the timeout the caller asked for is the time this call can take. + delivered &= awaitDelivery(delivery, Math.max(0, deadline - System.nanoTime())); + } + return delivered; + } + + private boolean awaitDelivery(Future delivery, long remainingNanos) { + try { + return Boolean.TRUE.equals(delivery.get(remainingNanos, TimeUnit.NANOSECONDS)); + } catch (TimeoutException e) { + // Left running rather than canceled: the events have been taken out of the buffer by + // now, so interrupting the delivery would only make losing them certain. + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (CancellationException e) { + // Not something the SDK's own processor does, but a custom one can hand back a future + // that is canceled, and that must not escape a call whose answer is a boolean. + return false; + } catch (ExecutionException e) { + Throwable cause = e.getCause() == null ? e : e.getCause(); + logger.error("Exception caught when flushing events: {}", LogValues.exceptionSummary(cause)); + logger.debug("{}", LogValues.exceptionTrace(cause)); + return false; + } + } + @VisibleForTesting void blockingFlush() { eventProcessor.blockingFlush(); diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClientInterface.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClientInterface.java index dd62ec5c9..0bdd02595 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClientInterface.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDClientInterface.java @@ -11,6 +11,7 @@ import java.io.Closeable; import java.util.Map; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; /** * The interface for the LaunchDarkly SDK client. @@ -142,10 +143,71 @@ public interface LDClientInterface extends Closeable { Future identify(LDContext context); /** - * Sends all pending events to LaunchDarkly. + * Writes down all pending events and sends them to LaunchDarkly. + *

+ * Recording an event does not on its own make it outlive the process. Where + * {@link com.launchdarkly.sdk.android.integrations.EventProcessorBuilder#eventPersistence( + * com.launchdarkly.sdk.android.integrations.EventPersistence)} is on, + * events are written in runs, so one recorded shortly before the process ends may never have been + * written at all; this call writes everything recorded so far, and the events then survive whether or + * not the delivery does. That write happens on the calling thread before this returns, at + * {@link com.launchdarkly.sdk.android.integrations.EventPersistence#DEFERRED} as well as at + * {@link com.launchdarkly.sdk.android.integrations.EventPersistence#IMMEDIATE}, so a call from the main + * thread touches the disk there. Where persistence is off, events live in memory only and nothing + * survives the process, whether or not this was called. + *

+ * That is what makes it worth calling where the process is about to end. An application that reports + * errors to LaunchDarkly and then crashes should flush from its uncaught exception handler, which runs + * while the process is still alive -- {@link #flushAndWait(long, TimeUnit)} is the better choice there, + * since it also waits for delivery. Terminations that run no application code, such as {@code SIGKILL}, + * a native crash, an ANR, or the system reclaiming a backgrounded process, cannot be covered this way, + * and they take whatever was recorded since the last write. */ void flush(); + /** + * Sends all pending events to LaunchDarkly and waits for them to be delivered. + *

+ * Unlike {@link #flush()}, which returns before the events reach the network, this reports + * whether they arrived, which is what makes it usable at a point where the application is about + * to lose the ability to send them: an uncaught exception handler, a move to the background, or + * any other last chance. Events buffered in memory do not survive the process, so a caller that + * knows the process is ending can use this to give them one. + * {@link com.launchdarkly.sdk.android.integrations.LDCrashHandler} does this for + * uncaught exceptions. + *

+ * It can only help while the process is still running code. An uncaught exception runs its + * handler first, and a move to the background is announced, so both leave time for this call. + * A {@code SIGKILL}, an ANR kill, a native crash, and the system reclaiming a backgrounded process + * run nothing at all, and events still in memory at that moment are lost whatever the + * application does. + *

+ * The timeout bounds the whole call, including when the SDK is configured for more than one + * environment. Choose it with the caller in mind: a dying process is not a good place to wait on + * a network request that may never answer. The call blocks the thread it is made on, so on the + * main thread the timeout also counts towards an ANR. On a network that does not answer, a + * delivery takes about 21 seconds to give up with the default HTTP configuration: two attempts a + * second apart, each allowed the timeout set by + * {@link com.launchdarkly.sdk.android.integrations.HttpConfigurationBuilder#connectTimeoutMillis(int)}. + * A shorter timeout returns {@code false} before then, so there it bounds the wait rather than + * reporting how the delivery went. + * + * @param timeout how long to wait for delivery + * @param unit the time unit of {@code timeout} + * @return true if the events were delivered, or there were none to deliver; false if the timeout + * expired first, the SDK is offline, closed, or otherwise unable to deliver them, or events + * recorded since the last time this was answered were lost on the way, by this delivery or an + * earlier one. Events the service refuses with an error that may pass, such as a 503, are not + * lost: they are retried a second later and again by every flush after that, so a later call + * can still deliver them, and where persistence is on so can a later run of the application. + * A 401 or 403 stops delivery for good. A {@code false} because the timeout expired does not + * mean the events were not sent: the delivery is left running when the caller stops waiting, + * and may still arrive if the process lives long enough. A caller that resends on + * {@code false} can therefore cause duplicates. + * @since 5.17.0 + */ + boolean flushAndWait(long timeout, TimeUnit unit); + /** * Returns a map of all feature flags for the current evaluation context. No events are sent to LaunchDarkly. * diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDFutures.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDFutures.java index c062d72eb..8bed20246 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDFutures.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/LDFutures.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -83,6 +84,28 @@ public static LDAwaitFuture fromFuture(Future future) { return result; } + /** + * Runs a blocking call on a pooled daemon thread and reports its result as a future. + *

+ * Use this where a caller has a deadline but the work it is waiting for has no way to take one. + * The call is left running if the caller stops waiting; nothing interrupts it. + * + * @param task the blocking call + * @param result type + * @return a future that completes with the call's result, or with whatever it threw + */ + public static Future fromBlockingCall(Callable task) { + LDAwaitFuture result = new LDAwaitFuture<>(); + getBridgeExecutor().execute(() -> { + try { + result.set(task.call()); + } catch (Throwable t) { + result.setException(t); + } + }); + return result; + } + /** * Returns a future that completes when the first of the given futures completes. * Equivalent to CompletableFuture.anyOf. Works with any {@link Future} (API-level safe). diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/OutboundEventBuffer.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/OutboundEventBuffer.java new file mode 100644 index 000000000..91ee8a894 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/OutboundEventBuffer.java @@ -0,0 +1,343 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.LogValues; +import com.launchdarkly.sdk.AttributeRef; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.internal.events.AggregatedEventSummarizer; +import com.launchdarkly.sdk.internal.events.Event; +import com.launchdarkly.sdk.internal.events.EventOutputFormatter; +import com.launchdarkly.sdk.internal.events.EventSummarizer; +import com.launchdarkly.sdk.internal.events.EventSummarizerInterface; +import com.launchdarkly.sdk.internal.events.EventsConfiguration; +import com.launchdarkly.sdk.internal.events.SameContextPerContextEventSummarizer; + +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * The counters and the encoder behind {@link DirectEventProcessor}: evaluations are folded into + * summary counters as they are recorded, and a flush turns a run of full-fidelity events, together + * with the counters accumulated beside them, into the payload they will be sent as. + *

+ * The full events are held by the processor rather than here. Capacity is counted once, against + * everything the SDK is holding, and the processor is the only place that can see all of it. + *

+ * The summarization and the wire format come from java-sdk-internal rather than being + * reimplemented here, so there is one definition of what an event looks like on the wire. + * {@code DefaultEventProcessor} is not reused along with them because it only accepts individual + * events and summarizes them itself, on the far side of the bounded queue this is meant to get in + * front of. + */ +final class OutboundEventBuffer { + private static final int INITIAL_OUTPUT_BUFFER_SIZE = 2000; + + private final EventOutputFormatter formatter; + private final EventSummarizerInterface summarizer; + private final LDLogger logger; + + /** + * How many distinct contexts may be counted between two drains, and which ones already are. + *

+ * The per-context summarizer keeps a counter set per context, keyed on the whole context with + * every attribute retained, and offers no way to ask how many it holds. Without this the only + * bound on it is how long the SDK goes without a drain -- and it is not drained at all while the + * client is offline, which is exactly when an application is free to go on evaluating. + */ + private final int maxContexts; + private final Set countedContexts; + /** + * The context counted last, compared by reference so that the set above is only hashed into when the + * context changes. {@link LDContext#hashCode()} is not cached, and an application evaluates against + * one instance until it identifies again. + */ + private LDContext lastCountedContext; + + /** + * Set when an event or a summary was dropped for failing to serialize, until the processor asks. + * Serialization runs on whichever thread commits, so this is the one piece of state here that is + * written outside the caller's lock. + */ + private final AtomicBoolean serializationFailed = new AtomicBoolean(false); + + /** + * @param allAttributesPrivate true to redact every context attribute except the key + * @param privateAttributes the individual context attributes to redact + * @param perContextSummarization true to emit one summary per context rather than one overall + * @param maxContexts how many distinct contexts may be counted between two drains + * @param logger where to report an event that cannot be serialized + */ + OutboundEventBuffer( + boolean allAttributesPrivate, + Collection privateAttributes, + boolean perContextSummarization, + int maxContexts, + LDLogger logger + ) { + // Only the private-attribute settings affect the output; the rest of EventsConfiguration + // describes the delivery behavior that the processor now handles itself, capacity included. + EventsConfiguration outputConfig = new EventsConfiguration(allAttributesPrivate, 0, + null, 0, null, null, 1, null, 0, false, false, privateAttributes, + perContextSummarization); + this.formatter = new EventOutputFormatter(outputConfig); + this.summarizer = perContextSummarization + ? new SameContextPerContextEventSummarizer() + : new AggregatedEventSummarizer(); + // One bucket overall, so there is no cardinality to bound and nothing to track it with. + this.maxContexts = perContextSummarization ? maxContexts : Integer.MAX_VALUE; + this.countedContexts = perContextSummarization ? new HashSet<>() : null; + this.logger = logger; + } + + /** + * Folds an evaluation into the summary counters, unless the evaluation asked to be left out of + * them. + *

+ * A counter is an aggregate rather than a buffered event, so no number of evaluations of a context + * already being counted can make this drop anything. What capacity does bound is how many distinct + * contexts are counted at once, because each one costs a retained context and its own counters. + *

+ * Not synchronized: the caller holds its own lock across this and the rest of the recording, as it + * does across {@link #takeSummaries}, and a second monitor inside it would only be taken for nothing. + * + * @param event the evaluation + * @return false if this event was not counted, because counting it would have meant holding + * a context beyond the configured capacity + */ + boolean summarize(Event.FeatureRequest event) { + // Checked here rather than in DirectEventProcessor, for the same reason the sampling ratio + // is: this is where an event arrives from outside. The processor summarizes from the values + // themselves, which have no such flag. java-sdk-internal's DefaultEventProcessor, which this + // path replaced, honored it, and a counter is the one thing no later stage can reconstruct. + if (event.isExcludeFromSummaries()) { + return true; + } + return summarize(event.getCreationDate(), event.getKey(), event.getVersion(), event.getVariation(), + event.getValue(), event.getDefaultVal(), event.getContext()); + } + + /** + * As {@link #summarize(Event.FeatureRequest)}, from the evaluation's values, so that an evaluation + * nobody needs a full event for costs no event to be built. + */ + boolean summarize(long timestamp, String flagKey, int flagVersion, int variation, LDValue value, + LDValue defaultValue, LDContext context) { + // Only a context that is not being counted yet can be turned away, so reaching the limit costs + // an application evaluating against one context nothing, however many evaluations it does. + if (countedContexts != null && context != lastCountedContext) { + if (!countedContexts.contains(context)) { + if (countedContexts.size() >= maxContexts) { + return false; + } + countedContexts.add(context); + } + lastCountedContext = context; + } + summarizer.summarizeEvent(timestamp, flagKey, flagVersion, variation, value, defaultValue, context); + return true; + } + + /** + * Serializes one event into the bytes it will be sent as, for a caller that keeps events + * somewhere other than this buffer's list. + *

+ * Serializing at record time rather than at flush is what lets an event be written somewhere that + * outlives the process: bytes can be appended to a log, an {@link Event} cannot. + *

+ * Deliberately not synchronized. It reads the formatter and writes only locals, so it shares + * nothing with a thread recording an event, and taking the lock would make an encode wait on a + * counter increment and the other way round. + * + * Sampling is not applied here. {@link DirectEventProcessor} draws for it once, as the event is + * recorded, and drawing again for the same event would square the odds against it. + * + * @param event the event + * @return the event's JSON object, or null if it could not be serialized + */ + byte[] serialize(Event event) { + return writeSingleObject(new Event[]{ event }, Collections.emptyList()); + } + + /** + * Takes the counters and forgets which contexts they were counted for. + *

+ * The two have to move together. Every path that resets the summarizer has to come through here, + * because one that reset the counters alone would spend the cardinality limit on contexts whose + * counters had already gone out, and the limit would never lift. + *

+ * Separate from {@link #serializeSummaries} so that the caller can take the counters in the same + * critical section it lifts the full events in. An evaluation writes a counter and a full event, + * and a commit that took the two at different moments could split one evaluation across two + * payloads. + * Not synchronized, for the reason {@link #summarize(Event.FeatureRequest)} is not. + */ + List takeSummaries() { + List summaries = summarizer.getSummariesAndReset(); + if (countedContexts != null) { + countedContexts.clear(); + lastCountedContext = null; + } + return summaries; + } + + /** + * Takes the counters and serializes them in one call, for a caller with nothing to coordinate the + * take with. + *

+ * Not for the commit path. That has to take the counters in the same critical section it takes the + * pending events in, or a commit can stage an evaluation's counter and leave its full event behind. + */ + List serializeSummariesAndReset() { + return serializeSummaries(takeSummaries()); + } + + /** + * Turns summary counters already taken from {@link #takeSummaries} into their summary events. + *

+ * Counters live only in memory, so whatever has not been summarized is what a crash takes with + * it. Summarizing at points where the events are about to be made durable is what bounds that + * loss, and it costs nothing in accuracy: LaunchDarkly sums the counters of every summary it + * receives, so several summaries covering the same evaluations count the same as one. + *

+ * A summary that cannot be serialized is dropped and logged rather than counted back in. Losing + * an aggregate of evaluations the SDK promised to report is bad, but everything the encoder can + * fail on is a property of the data it was handed -- the output stream is a byte array and cannot + * fail transiently -- so putting the counters back would make every later flush fail on the same + * summary while the counters behind it grew without bound. + *

+ * Deliberately not synchronized, for the reason {@link #serialize} is not: the counters are taken + * under the caller's lock and encoded outside it, so a thread recording an evaluation waits only + * for a reference swap and never for the encoder. + * + * @param summaries the counters taken for this commit + * @return one JSON object per summary, empty if there was nothing counted or nothing serialized + */ + List serializeSummaries(List summaries) { + if (summaries.isEmpty()) { + return Collections.emptyList(); + } + List serialized = new ArrayList<>(summaries.size()); + for (EventSummarizer.EventSummary summary : summaries) { + byte[] bytes = writeSingleObject(new Event[0], Collections.singletonList(summary)); + if (bytes != null) { + serialized.add(bytes); + } + } + return serialized; + } + + /** + * Serializes a run of events with one encoder instead of one per event. + *

+ * Each event still becomes its own JSON object, because the store frames them individually and a + * frame holding several objects could not be spliced into a payload. What the run shares is the + * output stream, the writer and the UTF-8 lookup, which {@link #writeSingleObject} otherwise + * allocates on every call. + *

+ * Deliberately not synchronized, and not sampling, for the reasons {@link #serialize} is neither. + * + * @param pending the events, in the order they were recorded + * @return one JSON object per event that serialized + */ + List serializeAll(List pending) { + if (pending.isEmpty()) { + return Collections.emptyList(); + } + List serialized = new ArrayList<>(pending.size()); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(INITIAL_OUTPUT_BUFFER_SIZE); + Writer writer = new BufferedWriter( + new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), INITIAL_OUTPUT_BUFFER_SIZE); + Event[] one = new Event[1]; + for (Event event : pending) { + one[0] = event; + byte[] bytes = writeSingleObject(one, Collections.emptyList(), + outputStream, writer); + if (bytes != null) { + serialized.add(bytes); + } + } + return serialized; + } + + /** + * Serializes exactly one output event and returns the JSON object on its own. + *

+ * {@link EventOutputFormatter} only offers to write a whole request body, which is a JSON array, + * and the single-event method beside it is private. So the array is written and its brackets are + * dropped, which is sound precisely because the count it returns says how many objects are in + * there: one object means the bytes between the brackets are that object. + * + * @return the object's bytes, or null if the formatter did not write exactly one event + */ + private byte[] writeSingleObject(Event[] events, List summaries) { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(INITIAL_OUTPUT_BUFFER_SIZE); + Writer writer = new BufferedWriter( + new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), INITIAL_OUTPUT_BUFFER_SIZE); + return writeSingleObject(events, summaries, outputStream, writer); + } + + /** + * @param outputStream reset before the write, so it may be shared across a run + * @param writer must wrap {@code outputStream}, and holds nothing buffered on entry + */ + private byte[] writeSingleObject(Event[] events, List summaries, + ByteArrayOutputStream outputStream, Writer writer) { + outputStream.reset(); + int written; + try { + written = formatter.writeOutputEvents(events, summaries, writer); + writer.flush(); + } catch (Exception e) { + serializationFailed.set(true); + logger.error("Dropping unserializable {}: {}", describe(events), + LogValues.exceptionSummary(e)); + logger.debug("{}", LogValues.exceptionTrace(e)); + // Whatever the writer buffered is pushed out and thrown away with the stream, so a failed + // event cannot bleed into the next one sharing this writer. + try { + writer.flush(); + } catch (Exception ignored) { + // The stream is reset by the next call either way. + } + outputStream.reset(); + return null; + } + if (written != 1) { + return null; + } + byte[] all = outputStream.toByteArray(); + if (all.length < 3 || all[0] != '[' || all[all.length - 1] != ']') { + // Not the shape this depends on. Refusing the event is the safe reading: a frame that is + // not one JSON object would corrupt every payload it was later spliced into. + serializationFailed.set(true); + logger.error("Dropping unserializable {}: the encoder did not produce a single JSON" + + " object", describe(events)); + return null; + } + return Arrays.copyOfRange(all, 1, all.length - 1); + } + + /** + * @return whether anything was dropped for failing to serialize since this was last called + */ + boolean takeSerializationFailure() { + return serializationFailed.getAndSet(false); + } + + /** Names what is being dropped, since this encoder is handed one event or one summary. */ + private static String describe(Event[] events) { + return events.length == 0 ? "summary event" + : "event of type " + events[0].getClass().getSimpleName(); + }} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/PlatformState.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/PlatformState.java index 6c66b28f2..d75e3903a 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/PlatformState.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/PlatformState.java @@ -55,4 +55,28 @@ interface ForegroundChangeListener { * @return a directory path */ File getCacheDir(); + + /** + * Returns the preferred filesystem location for files the SDK must not lose. + *

+ * This is not the cache directory: the system is free to delete cache files whenever it wants + * space, which is the opposite of what data written to survive a crash needs. It is also excluded + * from backups, so events recorded on one device are not restored onto another and reported as if + * they had happened there. + * + * @return a directory path + */ + File getNoBackupFilesDir(); + + /** + * Returns a name for the process the SDK is running in, stable across restarts of that process + * and different for each process of a multi-process application. + *

+ * An application can declare components in more than one process, and each process gets its own + * copy of the SDK with no memory shared between them. Anything the SDK keeps in files is shared + * whether it wants to be or not, so this is what lets each process keep its own. + * + * @return a process name safe to use as part of a filename + */ + String getProcessName(); } diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventPersistence.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventPersistence.java new file mode 100644 index 000000000..c853c8c18 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventPersistence.java @@ -0,0 +1,50 @@ +package com.launchdarkly.sdk.android.integrations; + +/** + * How far the SDK goes to make a recorded event outlive the process that recorded it. + *

+ * An event lives in memory until it is delivered, so a process that dies before the next flush takes + * everything recorded since the last one — including the crash an application was reporting when it died. + * Writing events to disk is what closes that gap: they are delivered on a later run instead of being lost. + * + * @see EventProcessorBuilder#eventPersistence(EventPersistence) + */ +public enum EventPersistence { + /** + * Events are kept in memory only, and nothing survives the process ending. + *

+ * The SDK does not touch the filesystem for events at all. Events that an earlier run with persistence + * turned on left on disk are not delivered, and stay there until persistence is turned back on. + */ + DISABLED, + + /** + * Events are written to disk, off the thread that recorded them. + *

+ * Recording stays as cheap as it is without persistence. An event from {@code track} or {@code identify} + * becomes durable a moment after it was recorded rather than immediately, and a process that dies inside + * that window still loses it. Evaluating a flag does not schedule a write of its own: its events wait in + * memory until enough of them have built up, or until the next {@code track}, {@code identify}, flush or + * delivery writes them. + *

+ * A flush is the exception. It writes on the calling thread, as {@link #IMMEDIATE} does, so that an + * application ending the process on purpose can rely on it. A flush on the main thread with + * {@link android.os.StrictMode} detecting disk writes therefore sees a violation. + */ + DEFERRED, + + /** + * Events are written to disk before {@code track} and {@code identify} return. + *

+ * The write happens on the calling thread, costing tens to hundreds of microseconds depending on the + * device and on how many evaluations are waiting to be encoded alongside it. In exchange there is no + * window: a process that dies the instant after {@code track} returns still reports that event on the + * next run. Evaluating a flag stays in memory regardless. + *

+ * Because this is disk I/O on the calling thread, an application that calls {@code track} or + * {@code identify} on the main thread with {@link android.os.StrictMode} detecting disk writes will see + * a violation for each call. Choose {@link #DEFERRED} for those callers, which keeps these writes off + * the calling thread. + */ + IMMEDIATE +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventProcessorBuilder.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventProcessorBuilder.java index 989222ae2..8ccbe93f6 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventProcessorBuilder.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/EventProcessorBuilder.java @@ -29,7 +29,7 @@ public abstract class EventProcessorBuilder implements ComponentConfigurer - * The default value is {@link #DEFAULT_CAPACITY}. + * The same number bounds how many distinct contexts flag evaluations are summarized for between two + * flushes. Evaluations for contexts beyond it are not counted until the next flush. + *

+ * The default value is {@link #DEFAULT_CAPACITY}. A capacity below one is treated as one; to + * stop sending events altogether, use {@link Components#noEvents()} instead. * * @param capacity the capacity of the event buffer * @return the builder */ public EventProcessorBuilder capacity(int capacity) { - this.capacity = capacity; + this.capacity = capacity < 1 ? 1 : capacity; return this; } @@ -134,6 +148,25 @@ public EventProcessorBuilder flushIntervalMillis(int flushIntervalMillis) { return this; } + /** + * Sets how far the SDK goes to make a recorded event outlive the process that recorded it. + *

+ * Anything other than {@link EventPersistence#DISABLED} appends events to a log under the + * application's no-backup files directory and delivers them on a later run, which is what lets the + * crash an application was reporting when it died reach LaunchDarkly at all. + * {@link EventPersistence#IMMEDIATE} additionally puts the write on the thread that called + * {@code track} or {@code identify}, so there is no window in which the event exists only in memory. + *

+ * The default value is {@link #DEFAULT_EVENT_PERSISTENCE}. + * + * @param eventPersistence how far to go to make events outlive the process + * @return the builder + */ + public EventProcessorBuilder eventPersistence(EventPersistence eventPersistence) { + this.eventPersistence = eventPersistence == null ? DEFAULT_EVENT_PERSISTENCE : eventPersistence; + return this; + } + /** * Marks a set of attribute names or subproperties as private. *

diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/LDCrashHandler.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/LDCrashHandler.java new file mode 100644 index 000000000..98e2bbb26 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/integrations/LDCrashHandler.java @@ -0,0 +1,126 @@ +package com.launchdarkly.sdk.android.integrations; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.VisibleForTesting; + +import com.launchdarkly.sdk.android.LDClient; +import com.launchdarkly.sdk.android.LDClientInterface; +import com.launchdarkly.sdk.android.LaunchDarklyException; + +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +/** + * An uncaught exception handler that gives the SDK its last chance to act before the process ends: + * it delivers the analytics events still in memory. + *

+ * Events are kept in memory until they are sent, so a crash loses the ones recorded since the last + * flush, and those are often the ones that explain it. An uncaught exception runs its handlers while + * the process is still alive, and this one uses that time to call + * {@link LDClientInterface#flushAndWait(long, TimeUnit)} for every configured environment before + * passing the exception on to the handler that was installed before it. + *

+ * Install it before your crash reporter: + * + * 


+ *     LDClient.init(application, config, context, 0);
+ *     LDCrashHandler.install(2, TimeUnit.SECONDS);
+ *     SentryAndroid.init(application, options -> { ... });
+ *
+ *

+ * Crash reporters install their own handler in front of the one they find and call it once they + * have saved their report, so installing this one first lets the report be saved before the events + * are sent. Installed after the crash reporter, this handler runs first and holds the report back + * for as long as the delivery takes, and a process that dies in that time loses the report as well. + *

+ * The timeout is how long the crash is held open for the events. The thread that threw waits for + * it, so when that is the main thread the application stays frozen until the events are delivered + * or the timeout expires; a couple of seconds is a reasonable budget. That is less than a delivery + * takes to give up on a network that does not answer, so on such a network the crash is passed on + * when the timeout expires. A delivery still running then is not canceled, and may finish while the + * rest of the handlers run. + *

+ * This handler only helps with uncaught Java and Kotlin exceptions. A {@code SIGKILL}, an ANR kill, + * a native crash, and the system reclaiming a backgrounded process run no handlers, so the events + * still in memory at that moment are lost. + *

+ * This class is not stable, and not subject to any backwards compatibility guarantees or semantic versioning. + * It is experimental. + * + * @since 5.17.0 + */ +public final class LDCrashHandler implements Thread.UncaughtExceptionHandler { + private static final Object installLock = new Object(); + + private final Callable client; + private final long timeoutMillis; + @Nullable + private final Thread.UncaughtExceptionHandler next; + + @VisibleForTesting + LDCrashHandler( + @NonNull Callable client, + long timeoutMillis, + @Nullable Thread.UncaughtExceptionHandler next + ) { + this.client = client; + this.timeoutMillis = timeoutMillis; + this.next = next; + } + + /** + * Makes this handler the default uncaught exception handler, in front of whichever handler was + * the default before. + *

+ * It can be called before {@link LDClient#init}: a crash that happens before the client exists + * has no events to deliver, and is passed on straight away. Calling it again while this handler + * is still the default does nothing, so the timeout given first is the one that applies. + * + * @param timeout how long a crash waits for the events to be delivered + * @param unit the time unit of {@code timeout} + */ + public static void install(long timeout, @NonNull TimeUnit unit) { + Objects.requireNonNull(unit, "unit"); + long timeoutMillis = Math.max(0, unit.toMillis(timeout)); + synchronized (installLock) { + Thread.UncaughtExceptionHandler previous = Thread.getDefaultUncaughtExceptionHandler(); + if (previous instanceof LDCrashHandler) { + return; + } + Thread.setDefaultUncaughtExceptionHandler( + new LDCrashHandler(LDClient::get, timeoutMillis, previous)); + } + } + + @Override + public void uncaughtException(@NonNull Thread thread, @NonNull Throwable throwable) { + try { + flush(); + } finally { + if (next != null) { + next.uncaughtException(thread, throwable); + } else { + throwable.printStackTrace(); + } + } + } + + private void flush() { + try { + LDClientInterface ldClient = client.call(); + if (ldClient == null) { + return; + } + // Waiting on the crashing thread is safe because the SDK enforces the timeout instead of + // trusting the delivery to finish, which also covers a crash that is the reason the + // delivery cannot complete. + ldClient.flushAndWait(timeoutMillis, TimeUnit.MILLISECONDS); + } catch (LaunchDarklyException notInitialized) { + // A crash before LDClient.init has no events to deliver. + } catch (Throwable ignored) { + // Nothing that goes wrong here is worth losing the crash over. + } + } +} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/EventProcessor.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/EventProcessor.java index d1c98af02..3ef5fadbe 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/EventProcessor.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/EventProcessor.java @@ -4,8 +4,10 @@ import com.launchdarkly.sdk.EvaluationReason; import com.launchdarkly.sdk.LDContext; import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.LDFutures; import java.io.Closeable; +import java.util.concurrent.Future; /** * Interface for an object that can send or store analytics events. @@ -90,7 +92,9 @@ void recordCustomEvent( * Specifies that any buffered events should be sent as soon as possible, rather than waiting * for the next flush interval. This method is asynchronous, so events still may not be sent * until a later time. However, calling {@link Closeable#close()} will synchronously deliver - * any events that were not yet delivered prior to shutting down. + * any events that were not yet delivered prior to shutting down, unless the SDK is offline. In + * that case nothing is sent; the events are kept for a later run if event persistence is on, and + * discarded otherwise. */ void flush(); @@ -98,4 +102,30 @@ void recordCustomEvent( * Specifies that any buffered events should be sent immediately, blocking until done. */ void blockingFlush(); + + /** + * Specifies that any buffered events should be sent immediately, and reports through the + * returned future whether they were delivered. + *

+ * This is the form the SDK itself uses, so that a caller with a deadline can wait for as long as + * it has and no longer, and so that several of these can be waited on together. Only the public + * API puts a timeout in a signature; see {@code LDClient.flushAndWait}. + *

+ * Nothing cancels the delivery when a caller stops waiting for it: by then the events have been + * taken out of the buffer, so interrupting the post would only make losing them certain. + * + * @return a future that completes with true if the events reached the service, or there were + * none to send; false if they could not be sent, including when a delivery that started + * earlier took them and then lost them + * @since 5.17.0 + */ + default Future flushAsync() { + // An implementation written before this method existed has only its unbounded blocking + // flush, so that runs on a thread of its own: the caller's deadline then bounds the wait + // rather than the flush, and the outcome it reports is still the flush's own. + return LDFutures.fromBlockingCall(() -> { + blockingFlush(); + return true; + }); + } } diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizer.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizer.java new file mode 100644 index 000000000..c5498e349 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizer.java @@ -0,0 +1,109 @@ +package com.launchdarkly.sdk.internal.events; + +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * {@link PerContextEventSummarizer} with a fast path for the context it saw last, and without the Java 8 + * library calls that Android lacks below API 24. + *

+ * The fast path is the reason this outlives the API-level fix. {@link LDContext#hashCode()} is not cached: + * every call flattens and hashes every attribute, so a map keyed on the context costs one full hash per + * evaluation, usually on the main thread. An application evaluates against the same {@code LDContext} + * instance until it identifies again, so comparing references first skips that hash on nearly every call. + *

+ * The API level: java-sdk-internal's summarizer calls {@code Map.computeIfAbsent} with a + * {@code java.util.function.Function}, neither of which exists before API 24. An application that has not + * turned on core library desugaring gets a {@code NoClassDefFoundError} at the first evaluation it + * summarizes, and lint cannot see it because the call is inside a library jar. + *

+ * A per-context summary can only be built through {@link EventSummarizer}'s package-private members, + * which is why this sits in java-sdk-internal's package rather than the SDK's. Not thread-safe; the + * caller serializes every call. + */ +public final class SameContextPerContextEventSummarizer implements EventSummarizerInterface { + private final Map summarizersByContext = new HashMap<>(); + + /** The context the last evaluation was counted for, compared by reference, and its summarizer. */ + private LDContext lastContext; + private EventSummarizer lastSummarizer; + + @Override + public void summarizeEvent( + long timestamp, + String flagKey, + int flagVersion, + int variation, + LDValue value, + LDValue defaultValue, + LDContext context + ) { + EventSummarizer summarizer; + if (context == lastContext && lastSummarizer != null) { + summarizer = lastSummarizer; + } else { + summarizer = summarizersByContext.get(context); + if (summarizer == null) { + summarizer = new EventSummarizer(context); + summarizersByContext.put(context, summarizer); + } + lastContext = context; + lastSummarizer = summarizer; + } + summarizer.summarizeEvent(timestamp, flagKey, flagVersion, variation, value, defaultValue, context); + } + + @Override + public List getSummariesAndReset() { + List summaries = new ArrayList<>(); + for (EventSummarizer summarizer : summarizersByContext.values()) { + EventSummarizer.EventSummary summary = summarizer.getSummaryAndReset(); + if (!summary.isEmpty()) { + summaries.add(summary); + } + } + summarizersByContext.clear(); + forgetLastContext(); + return summaries; + } + + @Override + public void restoreTo(List previousSummaries) { + summarizersByContext.clear(); + forgetLastContext(); + for (EventSummarizer.EventSummary summary : previousSummaries) { + if (summary.context != null && !summary.isEmpty()) { + EventSummarizer summarizer = new EventSummarizer(summary.context); + summarizer.restoreTo(summary); + summarizersByContext.put(summary.context, summarizer); + } + } + } + + @Override + public boolean isEmpty() { + for (EventSummarizer summarizer : summarizersByContext.values()) { + if (!summarizer.isEmpty()) { + return false; + } + } + return true; + } + + @Override + public void clear() { + summarizersByContext.clear(); + forgetLastContext(); + } + + /** Required wherever the map is cleared, or the fast path would count into a summarizer no longer in it. */ + private void forgetLastContext() { + lastContext = null; + lastSummarizer = null; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DiagnosticConfigTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DiagnosticConfigTest.java index 8243052b0..2806c65b2 100644 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DiagnosticConfigTest.java +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DiagnosticConfigTest.java @@ -73,7 +73,7 @@ public void customDiagnosticConfigurationEvents() throws Exception { .events( Components.sendEvents() .allAttributesPrivate(true) - .capacity(1000) + .capacity(3000) .diagnosticRecordingIntervalMillis(1_800_000) .flushIntervalMillis(60_000) ) @@ -83,11 +83,26 @@ public void customDiagnosticConfigurationEvents() throws Exception { ObjectBuilder expected = makeExpectedDefaults(); expected.put("allAttributesPrivate", true); expected.put("diagnosticRecordingIntervalMillis", 1_800_000); - expected.put("eventsCapacity", 1000); + expected.put("eventsCapacity", 3000); expected.put("eventsFlushIntervalMillis",60_000); Assert.assertEquals(expected.build(), diagnosticJson); } + @Test + public void diagnosticConfigurationReportsTheCapacityTheSdkWillUse() throws Exception { + // A capacity of nought or less runs at one, so that is what the service should be told. + for (int capacity : new int[] { 0, -5 }) { + LDConfig ldConfig = new LDConfig.Builder(AutoEnvAttributes.Disabled) + .events(Components.sendEvents().capacity(capacity)) + .build(); + + LDValue diagnosticJson = makeDiagnosticJson(ldConfig); + + Assert.assertEquals("capacity " + capacity, + 1, diagnosticJson.get("eventsCapacity").intValue()); + } + } + @Test public void customDiagnosticConfigurationStreaming() throws Exception { LDConfig ldConfig = new LDConfig.Builder(AutoEnvAttributes.Disabled) diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DirectEventProcessorTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DirectEventProcessorTest.java new file mode 100644 index 000000000..fe25cd11d --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DirectEventProcessorTest.java @@ -0,0 +1,2843 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.launchdarkly.sdk.EvaluationReason; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.integrations.EventPersistence; +import com.launchdarkly.sdk.android.integrations.EventProcessorBuilder; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; +import com.launchdarkly.sdk.android.subsystems.HttpConfiguration; +import com.launchdarkly.sdk.internal.events.DiagnosticStore; +import com.launchdarkly.sdk.internal.events.Event; +import com.launchdarkly.sdk.internal.events.EventSender; +import com.launchdarkly.testhelpers.httptest.HandlerSwitcher; +import com.launchdarkly.testhelpers.httptest.Handlers; +import com.launchdarkly.testhelpers.httptest.HttpServer; +import com.launchdarkly.testhelpers.httptest.RequestInfo; + +import org.junit.After; +import org.junit.Test; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Behavior of the SDK's own event processor, covering the parts that are not about buffering under + * load (see {@link EventProcessorBufferingTest} for those). + */ +public class DirectEventProcessorTest extends EventProcessorTestBase { + private static final String FLAG_KEY = "flag-key"; + private static final int FLAG_VERSION = 10; + private static final int VARIATION = 1; + private static final LDValue FLAG_VALUE = LDValue.of(true); + private static final LDValue DEFAULT_VALUE = LDValue.of(false); + + private static final int DEFAULT_CAPACITY = 100; + private static final String PAYLOAD_ID_HEADER = "X-LaunchDarkly-Payload-ID"; + + // Enough concurrent evaluations that close() reliably lands between the two writes one evaluation + // makes. Against the unfixed code this failed in the first trial of every run, by two to four + // events -- roughly the number of recorders that can sit in the gap at once. + private static final int RACE_TRIALS = 20; + private static final int RACE_RECORDERS = 4; + private static final int EVALUATIONS_PER_RACE_RECORDER = 2_000; + private static final int EVALUATIONS_BEFORE_CLOSE = 200; + private static final int RACE_CAPACITY = 30; + // Past anything the recorders can produce, so that a commit short of a feature event is short + // because the evaluation was split rather than because the buffer was full. + private static final int NO_DROP_CAPACITY = RACE_RECORDERS * EVALUATIONS_PER_RACE_RECORDER * 2; + private static final int SPLIT_TRIALS = 8; + + // Long enough that the only delivery in a test is the one it asks for. + private static final int NO_PERIODIC_FLUSH_MILLIS = 600_000; + // Short enough to keep the close tests quick; the production value is chosen for a real network. + private static final long CLOSE_BUDGET_MILLIS = 200; + + /** Created by makeEventProcessor, which the tests call instead of building a processor. */ + private final List diagnosticExecutors = new ArrayList<>(); + /** The commit executor the last processor made here commits on, for a test to wait behind. */ + private ExecutorService lastCommitExecutor; + + @After + public void shutDownDiagnosticExecutors() { + for (ExecutorService executor : diagnosticExecutors) { + executor.shutdownNow(); + } + } + + @Test + public void untrackedEvaluationProducesOnlyASummary() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + recordEvaluation(eventProcessor, false, null); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(1, events.size()); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void trackedEvaluationProducesAFeatureEventAndASummary() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, EvaluationReason.off(), DEFAULT_VALUE, true, null); + + List events = flushAndCollect(eventProcessor, server); + + LDValue featureEvent = requireEventOfKind(events, "feature"); + assertEquals(LDValue.of(FLAG_KEY), featureEvent.get("key")); + assertEquals(LDValue.of(VARIATION), featureEvent.get("variation")); + assertEquals(FLAG_VALUE, featureEvent.get("value")); + assertEquals(LDValue.of(FLAG_VERSION), featureEvent.get("version")); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void flushPersistsFeatureAndSummaryEventsSynchronouslyWhileOffline() throws Exception { + assumePersisting(); + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, EvaluationReason.off(), DEFAULT_VALUE, true, null); + eventProcessor.setOffline(true); + + eventProcessor.flush(); + + EventStore reader = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, "test", + DEFAULT_CAPACITY, true, logging.logger); + List persisted = reader.pendingEventPayloads(); + int featureEvents = 0; + int summaryEvents = 0; + for (byte[] payload : persisted) { + String kind = LDValue.parse(new String(payload, "UTF-8")).get("kind").stringValue(); + featureEvents += "feature".equals(kind) ? 1 : 0; + summaryEvents += "summary".equals(kind) ? 1 : 0; + } + assertEquals(1, featureEvents); + assertEquals(1, summaryEvents); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void immediatePersistencePutsATrackOnDiskBeforeItReturns() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).eventPersistence(EventPersistence.IMMEDIATE), true); + try { + // Offline so a delivery cannot drain the store out from under the assertion. The commit + // this is about runs either way; only the sending is held back. + eventProcessor.setOffline(true); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.of("data"), 2.5); + + // No waiting and no flush: the guarantee is that the call did the work before returning, which + // is the whole of what this setting buys and the only way a SIGKILL here still reports it. + assertEquals(1, kindsOnDisk("custom")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void deferredPersistenceStillPutsATrackOnDisk() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).eventPersistence(EventPersistence.DEFERRED), true); + try { + // Offline for the same reason as the immediate case: delivery would race the read. + eventProcessor.setOffline(true); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.of("data"), 2.5); + + // Durable a moment later rather than immediately: the commit was queued, not skipped. + long deadline = System.currentTimeMillis() + 5_000; + while (kindsOnDisk("custom") == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + assertEquals(1, kindsOnDisk("custom")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aDeferredTrackReachesTheDiskWhileADeliveryHoldsTheEventsThread() throws Exception { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + EventStore store = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, "test", + DEFAULT_CAPACITY, true, logging.logger); + DirectEventProcessor eventProcessor = makeEventProcessor(store, DEFAULT_CAPACITY, scheduler); + CountDownLatch deliveryAnswers = new CountDownLatch(1); + try { + // Stands in for a post to a network that does not answer, which holds the events thread for + // as long as its timeouts allow. + scheduler.submit(() -> awaitQuietly(deliveryAnswers, 10, TimeUnit.SECONDS)); + + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.of("data"), 2.5); + + long deadline = System.currentTimeMillis() + 5_000; + while (kindsOnDisk("custom") == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + assertEquals(1, kindsOnDisk("custom")); + } finally { + deliveryAnswers.countDown(); + eventProcessor.close(); + } + } + + @Test + public void aFlushThatWaitsWritesOnItsOwnThreadWhileADeliveryHoldsTheEventsThread() throws Exception { + // An evaluation is not a commit point, so a flush is what makes one durable. flushAsync hands + // the delivery to the events thread, which a post or a retry can hold for longer than the + // caller's timeout -- and flushAndWait is what an application calls when it is about to be + // killed, so the write cannot be left to that thread. + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + EventStore store = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, "test", + DEFAULT_CAPACITY, true, logging.logger); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + UNUSED_EVENTS_URI, null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY, true); + CountDownLatch deliveryAnswers = new CountDownLatch(1); + try { + eventProcessor.setOffline(false); + // Stands in for a post to a network that does not answer, which holds the events thread for + // as long as its timeouts allow. + scheduler.submit(() -> awaitQuietly(deliveryAnswers, 10, TimeUnit.SECONDS)); + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, EvaluationReason.off(), DEFAULT_VALUE, true, null); + + assertFalse("the delivery answered even though the events thread was held", + awaitFlush(eventProcessor, 0, TimeUnit.MILLISECONDS)); + + // Read with nothing waited on: the commit ran on the flushing thread, not the events one. + assertEquals(1, kindsOnDisk("feature")); + assertEquals(1, kindsOnDisk("summary")); + } finally { + deliveryAnswers.countDown(); + eventProcessor.close(); + } + } + + @Test + public void aDeferredFlushThatWaitsReachesTheDiskWhileADeliveryHoldsTheEventsThread() throws Exception { + // As above, where the application asked for writes to be queued rather than made on its own + // thread: a flush writes on the caller's thread regardless, so the events are on disk by the + // time it returns rather than whenever the commit executor gets to them. + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + EventStore store = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, "test", + DEFAULT_CAPACITY, true, logging.logger); + DirectEventProcessor eventProcessor = makeEventProcessor(store, DEFAULT_CAPACITY, scheduler); + CountDownLatch deliveryAnswers = new CountDownLatch(1); + try { + eventProcessor.setOffline(false); + scheduler.submit(() -> awaitQuietly(deliveryAnswers, 10, TimeUnit.SECONDS)); + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, EvaluationReason.off(), DEFAULT_VALUE, true, null); + + assertFalse("the delivery answered even though the events thread was held", + awaitFlush(eventProcessor, 0, TimeUnit.MILLISECONDS)); + + assertEquals(1, kindsOnDisk("feature")); + assertEquals(1, kindsOnDisk("summary")); + } finally { + deliveryAnswers.countDown(); + eventProcessor.close(); + } + } + + @Test + public void aDeferredFlushWritesWithoutWaitingForTheQueueATrackedEventsWriteGoesTo() throws Exception { + // A flush is what an application calls before ending the process on purpose, so at DEFERRED it + // writes on its own thread rather than queueing the write the way track does. Holding the commit + // executor keeps a queued write off the disk, so only a write on this thread can pass. + EventStore store = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, "test", + DEFAULT_CAPACITY, true, logging.logger); + DirectEventProcessor eventProcessor = makeEventProcessor(store, DEFAULT_CAPACITY, + EventUtil.makeEventsTaskExecutor()); + CountDownLatch commitsResume = new CountDownLatch(1); + try { + lastCommitExecutor.submit(() -> awaitQuietly(commitsResume, 10, TimeUnit.SECONDS)); + eventProcessor.recordCustomEvent(CONTEXT, "before-exit", null, null); + assertEquals(0, kindsOnDisk("custom")); + + eventProcessor.flush(); + + assertEquals(1, kindsOnDisk("custom")); + } finally { + commitsResume.countDown(); + eventProcessor.close(); + } + } + + @Test + public void aFailedDeliveryIsRetriedOnceUnderTheSamePayloadId() throws Exception { + try (HttpServer server = HttpServer.start(Handlers.sequential(Handlers.status(503), + Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + // A flush that waits, waits for the retry too, so it reports the delivery that worked. + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + RequestInfo failed = server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + RequestInfo retried = server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + assertEquals(failed.getHeader("X-LaunchDarkly-Payload-ID"), + retried.getHeader("X-LaunchDarkly-Payload-ID")); + assertEquals(failed.getBody(), retried.getBody()); + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void waitingToRetryDoesNotHoldTheEventsThread() throws Exception { + try (HttpServer server = HttpServer.start(Handlers.status(503))) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + EventStore store = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, "test", + DEFAULT_CAPACITY, true, logging.logger); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY); + try { + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + eventProcessor.flush(); + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + + // Well inside the retry delay: a thread sleeping through it would not get to this. + scheduler.submit(() -> { }).get(DirectEventProcessor.RETRY_DELAY_MILLIS / 2, + TimeUnit.MILLISECONDS); + + // One retry and no more; the batch then waits for the next flush. + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + server.getRecorder().requireNoRequests(DirectEventProcessor.RETRY_DELAY_MILLIS * 2, + TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aStoredBatchThisVersionCannotReadIsDroppedAndReportedWhileTheRestAreDelivered() throws Exception { + try (HttpServer server = HttpServer.start(Handlers.status(202))) { + EventStore store = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, "test", + DEFAULT_CAPACITY, true, logging.logger); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY); + try { + // Online, and past the startup listing, before the batch exists, so the flush below is the + // delivery that finds it rather than one that recovery or coming online starts, which + // nobody waits on. + eventProcessor.setOffline(false); + scheduler.submit(() -> { }).get(2, TimeUnit.SECONDS); + store.stage("{\"kind\":\"custom\",\"key\":\"damaged\"}".getBytes(StandardCharsets.UTF_8)); + EventStore.Batch damaged = closeOne(store); + assertNotNull(damaged); + // Its name still counts it, so it is listed without being read; the damage is only found on + // sending. + File damagedFile = findFile(eventsDirectory.getRoot(), "ready-" + damaged.payloadId + "-1"); + assertNotNull(damagedFile); + java.nio.file.Files.write(damagedFile.toPath(), + "not an event log".getBytes(StandardCharsets.UTF_8)); + eventProcessor.recordCustomEvent(CONTEXT, "intact", LDValue.ofNull(), null); + + assertFalse("a flush that lost a batch reported success", + awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + RequestInfo sent = server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + assertTrue(sent.getBody().contains("intact")); + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + assertFalse(damagedFile.exists()); + } finally { + eventProcessor.close(); + } + } + } + + /** Closes what a test expects to be exactly one batch, as anything short of a failed write leaves. */ + private static EventStore.Batch closeOne(EventStore store) { + List closed = store.closeBatch(); + assertEquals(1, closed.size()); + return closed.get(0); + } + + private static File findFile(File directory, String name) { + File[] files = directory.listFiles(); + if (files == null) { + return null; + } + for (File file : files) { + if (file.getName().equals(name)) { + return file; + } + File found = file.isDirectory() ? findFile(file, name) : null; + if (found != null) { + return found; + } + } + return null; + } + + /** @return how many events of this kind the store holds, read as another process would read it */ + private int kindsOnDisk(String kind) throws Exception { + EventStore reader = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, "test", + DEFAULT_CAPACITY, true, logging.logger); + int found = 0; + for (byte[] payload : reader.pendingEventPayloads()) { + if (kind.equals(LDValue.parse(new String(payload, "UTF-8")).get("kind").stringValue())) { + found++; + } + } + return found; + } + + @Test + public void persistenceIsOffUnlessTheApplicationAsksForIt() throws Exception { + try (HttpServer server = startEventsServer()) { + // Deliberately not eventsBuilder(), which turns persistence on: this is what an application + // gets without configuring anything. + EventProcessor eventProcessor = makeEventProcessor(server, + Components.sendEvents() + .capacity(DEFAULT_CAPACITY) + .flushIntervalMillis(NO_PERIODIC_FLUSH_MILLIS), + true); + try { + eventProcessor.recordIdentifyEvent(CONTEXT); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.of("data"), 2.5); + + EventStore reader = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, "test", + DEFAULT_CAPACITY, true, logging.logger); + assertTrue("an event reached the disk without being asked to", + reader.pendingEventPayloads().isEmpty()); + + // Held in memory instead, so turning persistence off costs durability and nothing else. + List events = flushAndCollect(eventProcessor, server); + requireEventOfKind(events, "identify"); + requireEventOfKind(events, "custom"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void identifyAndCustomEventsAreDelivered() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordIdentifyEvent(CONTEXT); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.of("data"), 2.5); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(2, events.size()); + requireEventOfKind(events, "identify"); + LDValue customEvent = requireEventOfKind(events, "custom"); + assertEquals(LDValue.of("an-event"), customEvent.get("key")); + assertEquals(LDValue.of("data"), customEvent.get("data")); + assertEquals(LDValue.of(2.5), customEvent.get("metricValue")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void unserializableMetricDoesNotPoisonLaterDeliveries() throws Exception { + try (HttpServer server = startEventsServer()) { + // One slot makes this cover both recovery and release of the poisoned event's capacity. + EventProcessor eventProcessor = makeEventProcessor(server, 1); + try { + // This is the event produced by LDClient.trackMetric(..., Double.NaN). Gson's + // strict writer rejects the non-finite metric. + eventProcessor.recordCustomEvent( + CONTEXT, "poison", LDValue.ofNull(), Double.NaN); + eventProcessor.blockingFlush(); + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + + eventProcessor.recordCustomEvent( + CONTEXT, "after-poison", LDValue.ofNull(), 1.0); + List events = flushAndCollect(eventProcessor, server); + + assertEquals(1, events.size()); + assertEquals(LDValue.of("after-poison"), requireEventOfKind(events, "custom").get("key")); + logging.assertErrorLogged("Dropping unserializable"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void unserializableMetricDoesNotDropSiblingEvents() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent( + CONTEXT, "poison", LDValue.ofNull(), Double.NaN); + eventProcessor.recordCustomEvent( + CONTEXT, "kept", LDValue.ofNull(), 1.0); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(1, events.size()); + assertEquals(LDValue.of("kept"), requireEventOfKind(events, "custom").get("key")); + logging.assertErrorLogged("Dropping unserializable"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void unserializableSummaryDoesNotPoisonLaterDeliveries() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + // Java callers can violate boolVariation's @NonNull contract. A null flag key can + // enter the summarizer, but Gson cannot use it as a JSON object member name. + eventProcessor.recordEvaluationEvent(CONTEXT, null, FLAG_VERSION, VARIATION, + FLAG_VALUE, null, DEFAULT_VALUE, false, null); + eventProcessor.blockingFlush(); + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + + recordEvaluation(eventProcessor, false, null); + List events = flushAndCollect(eventProcessor, server); + + assertEquals(1, events.size()); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + logging.assertErrorLogged("Dropping unserializable"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void unserializableSummaryDoesNotDropSiblingEvents() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordEvaluationEvent(CONTEXT, null, FLAG_VERSION, VARIATION, + FLAG_VALUE, null, DEFAULT_VALUE, false, null); + eventProcessor.recordIdentifyEvent(CONTEXT); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(1, countEventsOfKind(events, "identify")); + assertEquals(0, countEventsOfKind(events, "summary")); + logging.assertErrorLogged("Dropping unserializable"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void capacityLimitsFullEventsButNotSummaries() throws Exception { + try (HttpServer server = startEventsServer()) { + int capacity = 3; + EventProcessor eventProcessor = makeEventProcessor(server, capacity); + try { + for (int i = 0; i < capacity + 2; i++) { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + } + // Evaluations well past capacity still have to be counted in full, because a + // counter does not occupy a buffer slot. + int evaluations = capacity * 100; + for (int i = 0; i < evaluations; i++) { + recordEvaluation(eventProcessor, false, null); + } + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(capacity, countEventsOfKind(events, "custom")); + assertEquals(evaluations, summaryCountFor(events, FLAG_KEY)); + logging.assertWarnLogged("Exceeded event queue capacity"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aNonPositiveCapacityStillReportsRatherThanGoingSilent() throws Exception { + // Nought and negatives both mean one, and the processor and the buffer have to agree on + // that: disagreeing leaves the SDK holding an event it will never summarize, or summarizing + // for a buffer that will never send. Disabling events entirely is what noEvents() is for. + for (int capacity : new int[] { 0, -5 }) { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, capacity); + try { + recordEvaluation(eventProcessor, false, null); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals("capacity " + capacity + " summarized nothing", + 1, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + } + + @Test + public void closeDeliversBufferedEventsWithoutAnExplicitFlush() throws Exception { + // This is the case the SDK previously lost: a short session that records something and + // then shuts down before the periodic flush comes around. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + recordEvaluation(eventProcessor, false, null); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + eventProcessor.close(); + + List events = collectDelivered(server); + assertEquals(1, countEventsOfKind(events, "custom")); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + } + } + + @Test + public void closeNeverDeliversASummaryWithoutTheFeatureEventItCounted() throws Exception { + // An evaluation of a tracked flag writes a summary counter and a feature event. close() has + // to take both or neither. If it can take the counter and then refuse the event, it reports + // an evaluation that no event describes -- for experimentation traffic, a data point that + // disappears while the summary insists it happened. + // + // The window is the gap between those two writes, so this races them rather than asserting + // on a single ordering. Nothing here is timing-tolerant: the invariant holds for every + // interleaving, so any trial that breaks it is a real defect. + int featureEventsSeen = 0; + for (int trial = 0; trial < RACE_TRIALS; trial++) { + try (HttpServer server = startEventsServer()) { + // Deliberately small. The recorders outrun the buffer, so capacity is the only thing + // bounding the payload, and the test server records the body with a single unlooped + // read -- so anything past one socket read's worth comes back truncated, at a point + // that moves from run to run. A few kilobytes stays well clear of that. + DirectEventProcessor eventProcessor = + (DirectEventProcessor) makeEventProcessor(server, RACE_CAPACITY); + // Guarantees the final delivery has something in it, so collectDelivered always has + // a request to read even if every recorder loses the race. + eventProcessor.recordIdentifyEvent(CONTEXT); + + AtomicInteger recorded = new AtomicInteger(); + List recorders = new ArrayList<>(); + for (int i = 0; i < RACE_RECORDERS; i++) { + Thread recorder = new Thread(() -> { + for (int n = 0; n < EVALUATIONS_PER_RACE_RECORDER; n++) { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, true, null); + recorded.incrementAndGet(); + } + }); + recorders.add(recorder); + recorder.start(); + } + // Close in the middle of the burst rather than at the edge of it. Closing before the + // recorders are going would resolve the race trivially every time. + while (recorded.get() < EVALUATIONS_BEFORE_CLOSE) { + Thread.yield(); + } + eventProcessor.close(); + for (Thread recorder : recorders) { + recorder.join(); + } + + long dropped = eventProcessor.getAndClearDroppedCount(); + List events = collectDelivered(server); + int featureEvents = countEventsOfKind(events, "feature"); + // Every evaluation is accounted for one of three ways: delivered in full, dropped + // for capacity, or refused outright before anything was written. Only the first two + // may leave a counter behind, so a counter that matches neither is one that was + // taken from a half-recorded evaluation. + assertEquals("trial " + trial + ": a summary counted an evaluation whose feature" + + " event was neither delivered nor dropped", + featureEvents + dropped, summaryCounters(events, FLAG_KEY)); + featureEventsSeen += featureEvents; + } + } + // Guards against the whole thing passing because nothing ever got as far as being delivered. + assertTrue("no evaluation survived to be delivered, so nothing was actually compared", + featureEventsSeen > 0); + } + + /** + * Like {@code summaryCountFor}, but returns zero rather than failing when no summary was + * delivered. Here a trial in which close() beat every recorder is a legitimate outcome. + */ + private int summaryCounters(List events, String flagKey) { + int total = 0; + for (LDValue event : events) { + if (!"summary".equals(event.get("kind").stringValue())) { + continue; + } + for (LDValue counter : event.get("features").get(flagKey).get("counters").values()) { + total += counter.get("count").intValue(); + } + } + return total; + } + + @Test + public void eventsRecordedWhileOfflineAreRetainedAndSentOnceOnline() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.setOffline(true); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + recordEvaluation(eventProcessor, false, null); + + eventProcessor.blockingFlush(); + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + + eventProcessor.setOffline(false); + List events = flushAndCollect(eventProcessor, server); + + assertEquals(1, countEventsOfKind(events, "custom")); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void closingWhileOfflineStaysOffTheNetwork() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + eventProcessor.setOffline(true); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + recordEvaluation(eventProcessor, false, null); + + eventProcessor.close(); + + server.getRecorder().requireNoRequests(500, TimeUnit.MILLISECONDS); + } + } + + @Test + public void goingOnlineForTheFirstTimeDoesNotDeliverOnItsOwn() throws Exception { + // What LDClient does at startup: the initial identify is recorded while the processor is + // still offline, and initialization then turns it on. The identify waits to be batched with + // what follows rather than going out alone. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = buildOfflineEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY), true); + try { + eventProcessor.recordIdentifyEvent(CONTEXT); + + eventProcessor.setOffline(false); + + server.getRecorder().requireNoRequests(500, TimeUnit.MILLISECONDS); + List events = flushAndCollect(eventProcessor, server); + assertEquals(1, countEventsOfKind(events, "identify")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void closeCancelsThePeriodicFlushAndNothingAfterItSchedulesAnother() throws Exception { + List> scheduled = Collections.synchronizedList(new ArrayList<>()); + ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1) { + @Override + public ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, + long delay, TimeUnit unit) { + ScheduledFuture future = super.scheduleWithFixedDelay(command, initialDelay, delay, unit); + scheduled.add(future); + return future; + } + }; + // Otherwise shutting the executor down would cancel the task for close(), and this could not + // tell whether close() did. + scheduler.setContinueExistingPeriodicTasksAfterShutdownPolicy(true); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + NO_PERIODIC_FLUSH_MILLIS, scheduler); + try { + eventProcessor.setOffline(false); + eventProcessor.close(); + eventProcessor.setOffline(true); + eventProcessor.setOffline(false); + eventProcessor.setInBackground(true); + eventProcessor.setInBackground(false); + + assertEquals(1, scheduled.size()); + assertTrue("close() left the periodic flush running", scheduled.get(0).isCancelled()); + } finally { + scheduler.shutdownNow(); + } + } + + @Test + public void anOutageDoesNotRestartTheFlushInterval() throws Exception { + // Restarting it on every reconnect would let a run of brief outages hold events back for + // far longer than one interval. + AtomicInteger scheduled = new AtomicInteger(); + ScheduledExecutorService scheduler = new ScheduledThreadPoolExecutor(1) { + @Override + public ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, + long delay, TimeUnit unit) { + scheduled.incrementAndGet(); + return super.scheduleWithFixedDelay(command, initialDelay, delay, unit); + } + }; + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + NO_PERIODIC_FLUSH_MILLIS, scheduler); + try { + eventProcessor.setOffline(false); + for (int i = 0; i < 5; i++) { + eventProcessor.setOffline(true); + eventProcessor.setOffline(false); + } + + assertEquals(1, scheduled.get()); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void eventsOnTheirWayIntoTheStoreStillCountAgainstCapacity() throws Exception { + // A commit takes the pending events and encodes them before staging, and for that long they + // are in neither place. Counting only the two would let an event recorded then past capacity. + CountDownLatch staging = new CountDownLatch(1); + CountDownLatch releaseStaging = new CountDownLatch(1); + // Persisting, since without it a commit point queues no commit at all. + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", 1, true, + logging.logger) { + @Override + boolean stageReserved(byte[] serializedEvent) { + staging.countDown(); + awaitQuietly(releaseStaging, 5, TimeUnit.SECONDS); + return super.stageReserved(serializedEvent); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(store, 1, scheduler); + try { + eventProcessor.recordIdentifyEvent(CONTEXT); + assertTrue("the commit never started staging", staging.await(2, TimeUnit.SECONDS)); + + eventProcessor.recordIdentifyEvent(CONTEXT); + releaseStaging.countDown(); + + assertEquals(1, eventProcessor.getAndClearDroppedCount()); + } finally { + releaseStaging.countDown(); + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void anEventAlreadyStagedIsNotCountedTwiceAgainstCapacity() throws Exception { + // Paused just after the first event reached the store, with the commit that staged it still + // running. Counted once, it leaves room for a second event under a capacity of two; counted both + // as staged and as on its way, it would not. + CountDownLatch staged = new CountDownLatch(1); + CountDownLatch releaseCommit = new CountDownLatch(1); + // Persisting, for the reason the test above is. + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", 2, true, + logging.logger) { + @Override + boolean stageReserved(byte[] serializedEvent) { + boolean result = super.stageReserved(serializedEvent); + staged.countDown(); + awaitQuietly(releaseCommit, 5, TimeUnit.SECONDS); + return result; + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(store, 2, scheduler); + try { + eventProcessor.recordIdentifyEvent(CONTEXT); + assertTrue("the commit never staged the event", staged.await(2, TimeUnit.SECONDS)); + + eventProcessor.recordIdentifyEvent(CONTEXT); + releaseCommit.countDown(); + + assertEquals(0, eventProcessor.getAndClearDroppedCount()); + } finally { + releaseCommit.countDown(); + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void theStoreAsksThePlatformWhereItLivesOffTheCallersThread() throws Exception { + // LDClient.init builds the processor, usually on the main thread, and answering touches the disk: + // the platform creates the no-backup directory when asked for it, and before API 28 the process + // name is read from /proc. + Queue askedOn = new ConcurrentLinkedQueue<>(); + MockPlatformState platformState = new MockPlatformState() { + @Override + public File getNoBackupFilesDir() { + askedOn.add(Thread.currentThread()); + return eventsDirectory.getRoot(); + } + + @Override + public String getProcessName() { + askedOn.add(Thread.currentThread()); + return "test"; + } + }; + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = buildOfflineEventProcessor(server, + Components.sendEvents().eventPersistence(EventPersistence.DEFERRED) + .flushIntervalMillis(NO_PERIODIC_FLUSH_MILLIS), + true, platformState); + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + eventProcessor.flush(); + eventProcessor.blockingFlush(); + eventProcessor.close(); + + assertFalse("the platform was never asked, so this proves nothing", askedOn.isEmpty()); + assertFalse("the platform was asked on the caller's thread", + askedOn.contains(Thread.currentThread())); + } + } + + @Test + public void withoutPersistenceTheStoreNeverAsksThePlatformWhereItLives() throws Exception { + // Asking is itself a filesystem operation, so a store that never asks never touches the disk: + // not for a previous run's events at startup, and not on every delivery after. + AtomicInteger asked = new AtomicInteger(); + MockPlatformState platformState = new MockPlatformState() { + @Override + public File getNoBackupFilesDir() { + asked.incrementAndGet(); + return eventsDirectory.getRoot(); + } + + @Override + public String getProcessName() { + asked.incrementAndGet(); + return "test"; + } + }; + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = buildOfflineEventProcessor(server, + Components.sendEvents().eventPersistence(EventPersistence.DISABLED) + .flushIntervalMillis(NO_PERIODIC_FLUSH_MILLIS), + true, platformState); + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + eventProcessor.flush(); + eventProcessor.blockingFlush(); + eventProcessor.close(); + + assertEquals("the event was never delivered, so this proves nothing", + 1, countEventsOfKind(collectDelivered(server), "custom")); + assertEquals("a store without persistence looked for its directory", 0, asked.get()); + } + } + + @Test + public void concurrentImmediateTracksShareCommitsAndEachIsDurableOnReturn() throws Exception { + final int trackers = 8; + Queue staged = new ConcurrentLinkedQueue<>(); + Queue durable = new ConcurrentLinkedQueue<>(); + AtomicInteger commits = new AtomicInteger(); + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", DEFAULT_CAPACITY, true, + logging.logger) { + @Override + boolean stageReserved(byte[] serializedEvent) { + staged.add(new String(serializedEvent, StandardCharsets.UTF_8)); + return super.stageReserved(serializedEvent); + } + + @Override + void commit() { + commits.incrementAndGet(); + // A slow disk, so that the other tracks arrive while this write is under way. + sleepQuietly(50); + // Everything staged so far was staged under the commit lock this commit holds. + List writing = new ArrayList<>(staged); + super.commit(); + durable.addAll(writing); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), UNUSED_EVENTS_URI, + null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, scheduler, diagnosticExecutor, + store, DEFAULT_CAPACITY, true); + try { + CountDownLatch start = new CountDownLatch(1); + AtomicInteger notDurableOnReturn = new AtomicInteger(); + List threads = new ArrayList<>(); + for (int i = 0; i < trackers; i++) { + String key = "event-" + i; + Thread thread = new Thread(() -> { + awaitQuietly(start, 5, TimeUnit.SECONDS); + eventProcessor.recordCustomEvent(CONTEXT, key, LDValue.ofNull(), null); + boolean found = false; + for (String event : durable) { + found |= event.contains("\"key\":\"" + key + "\""); + } + if (!found) { + notDurableOnReturn.incrementAndGet(); + } + }); + thread.start(); + threads.add(thread); + } + start.countDown(); + for (Thread thread : threads) { + thread.join(10_000); + } + + assertEquals("a track returned before its event was on disk", 0, notDurableOnReturn.get()); + assertTrue("each track committed on its own: " + commits.get() + " commits for " + + trackers + " tracks", commits.get() < trackers); + } finally { + eventProcessor.close(); + } + } + + private static void sleepQuietly(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @Test + public void deferredPersistenceWritesOnTheCallersThreadOnlyForAFlush() throws Exception { + Queue committedOn = new ConcurrentLinkedQueue<>(); + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", DEFAULT_CAPACITY, true, + logging.logger) { + @Override + void commit() { + committedOn.add(Thread.currentThread()); + super.commit(); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(store, DEFAULT_CAPACITY, scheduler); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + eventProcessor.recordCustomEvent(CONTEXT, "another", LDValue.ofNull(), null); + lastCommitExecutor.submit(() -> { }).get(2, TimeUnit.SECONDS); + + assertFalse("nothing was ever committed, so this proves nothing", committedOn.isEmpty()); + assertFalse("a track committed on the caller's thread", + committedOn.contains(Thread.currentThread())); + + eventProcessor.recordCustomEvent(CONTEXT, "before-exit", LDValue.ofNull(), null); + eventProcessor.flush(); + + assertTrue("a flush left its commit to another thread", + committedOn.contains(Thread.currentThread())); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void aFullPendingRunIsCommittedWherePersistenceIsOn() throws Exception { + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", DEFAULT_CAPACITY, true, + logging.logger); + + assertTrue("the run was never committed", pendingEventsInStoreAfterAFullRun(store) > 0); + } + + @Test + public void aFullPendingRunWaitsForTheFlushWherePersistenceIsOff() throws Exception { + // Committing it would make nothing durable, and would put the encode in among the evaluations. + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", DEFAULT_CAPACITY, false, + logging.logger); + + assertEquals(0, pendingEventsInStoreAfterAFullRun(store)); + } + + @Test + public void aBatchDoesNotCountWhileItsRequestIsOutAndCountsAgainIfTheSendIsToBeRetried() + throws Exception { + for (boolean persistEvents : new boolean[] { true, false }) { + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", 1, persistEvents, + logging.logger); + assertTrue(store.stage(serialized("first"))); + EventStore.Batch batch = closeOne(store); + + store.startedSending(batch); + assertEquals(0, store.getPendingEventCount()); + assertTrue(store.stage(serialized("while-sending"))); + + store.stoppedSending(batch); + assertEquals(2, store.getPendingEventCount()); + assertFalse(store.stage(serialized("after-retryable"))); + } + } + + @Test + public void aSentBatchLeavesTheCountOnlyOnceWhenItIsDelivered() throws Exception { + for (boolean persistEvents : new boolean[] { true, false }) { + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", 2, persistEvents, + logging.logger); + assertTrue(store.stage(serialized("first"))); + EventStore.Batch batch = closeOne(store); + assertTrue(store.stage(serialized("second"))); + + store.startedSending(batch); + store.remove(batch); + assertEquals(1, store.getPendingEventCount()); + store.stoppedSending(batch); + assertEquals(1, store.getPendingEventCount()); + store.pendingBatches(); + assertEquals(1, store.getPendingEventCount()); + } + } + + /** + * What the contract tests' "buffer is reset after flush" does: they have the payload as soon as it is + * sent, and record the next event before the SDK has heard back. + */ + @Test + public void aFullStoreMakesRoomAsSoonAsItsBatchIsSent() throws Exception { + CountDownLatch received = new CountDownLatch(1); + CountDownLatch respond = new CountDownLatch(1); + try (HttpServer server = HttpServer.start(Handlers.sequential( + context -> { + received.countDown(); + awaitQuietly(respond, 5, TimeUnit.SECONDS); + context.setStatus(202); + }, + Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, 2); + try { + eventProcessor.recordCustomEvent(CONTEXT, "first", LDValue.ofNull(), null); + eventProcessor.recordCustomEvent(CONTEXT, "second", LDValue.ofNull(), null); + eventProcessor.flush(); + assertTrue("the first batch was never sent", received.await(5, TimeUnit.SECONDS)); + + eventProcessor.recordCustomEvent(CONTEXT, "one-more", LDValue.ofNull(), null); + respond.countDown(); + eventProcessor.flush(); + + List keys = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + RequestInfo request = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + for (LDValue event : LDValue.parse(request.getBody()).values()) { + keys.add(event.get("key").stringValue()); + } + } + assertEquals(Arrays.asList("first", "second", "one-more"), keys); + } finally { + eventProcessor.close(); + } + } + } + + private static byte[] serialized(String key) { + return ("{\"kind\":\"custom\",\"key\":\"" + key + "\"}").getBytes(StandardCharsets.UTF_8); + } + + /** + * Records more tracked evaluations than a pending run holds, lets the commit thread run whatever that + * queued, and reports how many events reached the store. + */ + private int pendingEventsInStoreAfterAFullRun(EventStore store) throws Exception { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(store, DEFAULT_CAPACITY, scheduler); + try { + for (int i = 0; i < 40; i++) { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, EvaluationReason.off(), DEFAULT_VALUE, true, null); + } + lastCommitExecutor.submit(() -> { }).get(2, TimeUnit.SECONDS); + return store.getPendingEventCount(); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void closeStillWritesWhenTheEventsThreadNeverGetsToIt() throws Exception { + // The events thread is single-threaded, so a delivery that hangs would hold a final commit queued + // there for longer than close() waits. The write is close()'s promise, so it does not run there. + AtomicBoolean closing = new AtomicBoolean(false); + Queue committedDuringClose = new ConcurrentLinkedQueue<>(); + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", DEFAULT_CAPACITY, true, + logging.logger) { + @Override + void commit() { + if (closing.get()) { + committedDuringClose.add(Thread.currentThread()); + } + super.commit(); + } + }; + CountDownLatch releaseEventsThread = new CountDownLatch(1); + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + UNUSED_EVENTS_URI, null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY); + try { + Thread[] eventsThread = new Thread[1]; + CountDownLatch eventsThreadHeld = new CountDownLatch(1); + scheduler.submit(() -> { + eventsThread[0] = Thread.currentThread(); + eventsThreadHeld.countDown(); + awaitQuietly(releaseEventsThread, 5, TimeUnit.SECONDS); + }); + assertTrue(eventsThreadHeld.await(5, TimeUnit.SECONDS)); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + closing.set(true); + eventProcessor.close(); + + assertFalse("close() returned without writing the event down", + committedDuringClose.isEmpty()); + assertFalse("the final commit waited on the events thread", + committedDuringClose.contains(eventsThread[0])); + assertEquals(1, store.getPendingEventCount()); + } finally { + releaseEventsThread.countDown(); + scheduler.shutdownNow(); + } + } + + @Test + public void eventsHeldDuringAnOutageGoOutWithTheNextPeriodicFlush() throws Exception { + // Analytics go out through AnalyticsEventSender rather than the injectable one, so the + // delivery has to be watched at the server rather than at the stub. + try (HttpServer server = startEventsServer()) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, 50, 60_000, DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, + scheduler); + try { + eventProcessor.setOffline(true); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + server.getRecorder().requireNoRequests(300, TimeUnit.MILLISECONDS); + + eventProcessor.setOffline(false); + + List events = collectDelivered(server); + assertEquals(1, countEventsOfKind(events, "custom")); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + } + + @Test + public void debugEventIsSentWhileDebuggingIsActive() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + long debugUntil = System.currentTimeMillis() + 3_600_000; + recordEvaluation(eventProcessor, false, debugUntil); + + List events = flushAndCollect(eventProcessor, server); + + LDValue debugEvent = requireEventOfKind(events, "debug"); + assertEquals(LDValue.of(FLAG_KEY), debugEvent.get("key")); + // A debug event carries the full context rather than just its keys. + assertEquals(LDValue.of(CONTEXT.getKey()), debugEvent.get("context").get("key")); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void debugEventIsNotSentOnceDebuggingHasExpired() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + long debugUntil = System.currentTimeMillis() - 3_600_000; + recordEvaluation(eventProcessor, false, debugUntil); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(0, countEventsOfKind(events, "debug")); + assertEquals(1, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void recordingAfterCloseIsIgnored() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + eventProcessor.recordCustomEvent(CONTEXT, "before-close", LDValue.ofNull(), null); + eventProcessor.close(); + collectDelivered(server); + + eventProcessor.recordCustomEvent(CONTEXT, "after-close", LDValue.ofNull(), null); + eventProcessor.flush(); + + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + } + } + + @Test + public void diagnosticInitEventIsSentWhenComingOnline() throws Exception { + try (HttpServer server = startEventsServer()) { + // makeEventProcessor takes the processor online, which is what triggers the init event. + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY, false); + try { + RequestInfo request = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + + assertEquals("/mobile/events/diagnostic", request.getPath()); + LDValue body = LDValue.parse(request.getBody()); + assertEquals(LDValue.of("diagnostic-init"), body.get("kind")); + // Only one, even though going online and coming to the foreground both ask for it. + returnToTheForegroundWithTheDiagnosticsThreadFree(eventProcessor); + server.getRecorder().requireNoRequests(500, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void diagnosticInitEventIsNotSentWhileOffline() throws Exception { + // Built offline and left that way. Waiting for the init first, as this used to, means the + // only thing stopping a second one is that the first already went -- so the test passes just + // as happily against a processor that sends init events while offline. + BlockingQueue posted = new LinkedBlockingQueue<>(); + EventSender sender = new StubEventSender() { + @Override + public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) { + posted.add(LDValue.parse(new String(data, StandardCharsets.UTF_8))); + return new Result(true, false, null); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(sender, makeDiagnosticStore(), + NO_PERIODIC_FLUSH_MILLIS, scheduler); + try { + assertNull("a diagnostic event was sent while offline", + posted.poll(500, TimeUnit.MILLISECONDS)); + + // Proof that the silence above was the offline state and not a fixture that could never + // have sent anything. + eventProcessor.setOffline(false); + assertEquals(LDValue.of("diagnostic-init"), requirePosted(posted).get("kind")); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void beingToldToShutDownStopsRecordingAndDelivery() throws Exception { + // A 401 means the mobile key will not start working again, so the SDK is supposed to stop + // for the life of the process rather than keep posting events nobody will accept. + try (HttpServer server = HttpServer.start(Handlers.status(401))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "before", LDValue.ofNull(), null); + eventProcessor.blockingFlush(); + server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + + eventProcessor.recordCustomEvent(CONTEXT, "after", LDValue.ofNull(), null); + eventProcessor.blockingFlush(); + + server.getRecorder().requireNoRequests(500, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void flushWithTimeoutReportsDeliveredEvents() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + assertEquals(1, countEventsOfKind(collectDelivered(server), "custom")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aCommitNeverSplitsAnEvaluationAcrossTwoCommits() throws Exception { + // The other half of the atomicity invariant. close() only ever commits once, so it can show + // an evaluation being stranded but not one being split: a counter staged by commit N with its + // feature event following in N+1. Payloads cannot show it here, because a delivery reads the + // store and may cut or join commits for its own reasons, so this looks at what each commit + // stages instead, while recording continues. + // + // Every evaluation is tracked and the capacity is far beyond what the run produces, so within + // a commit the counter for the flag and the number of feature events are the same number. + // + // Persistence is off so that the store never commits on its own; every commit() is one of the + // processor's. The processor stays offline, so each flush commits and delivers nothing. + // + // Repeated because the window is narrow -- the two writes are adjacent, and a commit has to + // land between them. + int commitsWithCounters = 0; + for (int trial = 0; trial < SPLIT_TRIALS; trial++) { + List> commits = new ArrayList<>(); + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", NO_DROP_CAPACITY, + false, logging.logger) { + private List staged = new ArrayList<>(); + + @Override + synchronized boolean stage(byte[] serializedEvent, boolean bypassingCapacity) { + staged.add(LDValue.parse(new String(serializedEvent, StandardCharsets.UTF_8))); + return super.stage(serializedEvent, bypassingCapacity); + } + + @Override + synchronized boolean stageReserved(byte[] serializedEvent) { + staged.add(LDValue.parse(new String(serializedEvent, StandardCharsets.UTF_8))); + return super.stageReserved(serializedEvent); + } + + @Override + synchronized void commit() { + commits.add(staged); + staged = new ArrayList<>(); + super.commit(); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(store, NO_DROP_CAPACITY, scheduler); + try { + AtomicInteger recorded = new AtomicInteger(); + List recorders = new ArrayList<>(); + for (int i = 0; i < RACE_RECORDERS; i++) { + Thread recorder = new Thread(() -> { + for (int n = 0; n < EVALUATIONS_PER_RACE_RECORDER; n++) { + recordEvaluation(eventProcessor, true, null); + recorded.incrementAndGet(); + } + }); + recorders.add(recorder); + recorder.start(); + } + int target = RACE_RECORDERS * EVALUATIONS_PER_RACE_RECORDER; + while (recorded.get() < target) { + eventProcessor.blockingFlush(); + } + for (Thread recorder : recorders) { + recorder.join(); + } + eventProcessor.blockingFlush(); + + assertEquals("trial " + trial + ": events were dropped, so a commit may be short" + + " for that reason instead", + 0, eventProcessor.getAndClearDroppedCount()); + int counted = 0; + synchronized (store) { + for (List commit : commits) { + int counters = summaryCounters(commit, FLAG_KEY); + assertEquals("trial " + trial + ": a commit counted evaluations whose feature" + + " events were staged separately", + countEventsOfKind(commit, "feature"), counters); + counted += counters; + if (counters > 0) { + commitsWithCounters++; + } + } + } + assertEquals("trial " + trial + ": some evaluations were never committed", + target, counted); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + // Otherwise a single commit per trial would satisfy everything above without a commit ever + // having overlapped a recording. + assertTrue("every evaluation went out in one commit, so nothing was interleaved", + commitsWithCounters > SPLIT_TRIALS); + } + + @Test + public void periodicTaskSurvivesErrorFromSender() throws Exception { + // An Error (not Exception) from a scheduled run used to cancel the repeating future with + // nothing logged, after which enableOrDisableTask kept returning that dead future forever. + // Driven through diagnostics, because that is the periodic task the injectable EventSender + // still carries once analytics go out through the store and AnalyticsEventSender. + CountDownLatch firstPeriodicAttempt = new CountDownLatch(1); + BlockingQueue delivered = new LinkedBlockingQueue<>(); + EventSender sender = new StubEventSender() { + private final AtomicInteger periodicAttempts = new AtomicInteger(); + + @Override + public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) { + String kind = LDValue.parse(new String(data, StandardCharsets.UTF_8)) + .get("kind").stringValue(); + if ("diagnostic-init".equals(kind)) { + return new Result(true, false, null); // one-shot, not the series under test + } + if (periodicAttempts.getAndIncrement() == 0) { + firstPeriodicAttempt.countDown(); + throw new Error("periodic diagnostics"); + } + delivered.add(data); + return new Result(true, false, null); + } + }; + + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(sender, + URI.create("https://events.example"), makeDiagnosticStore(), + NO_PERIODIC_FLUSH_MILLIS, 40, DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, + scheduler); + try { + // Coming online is the only thing that schedules the series. Nothing after this point + // touches the processor's state, so it has to stay alive on its own. + eventProcessor.setOffline(false); + assertTrue("sender never saw the first periodic diagnostic", + firstPeriodicAttempt.await(2, TimeUnit.SECONDS)); + + assertNotNull("periodic diagnostics did not run again after Error", + delivered.poll(2, TimeUnit.SECONDS)); + logging.assertErrorLogged("Unexpected error in event processor"); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void flushWithTimeoutReportsSuccessWhenThereIsNothingToSend() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + // Nothing was recorded, so the caller's events are not waiting anywhere. + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void unexpectedRecordingErrorDoesNotBubbleToCallerAndLogs() throws Exception { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + NO_PERIODIC_FLUSH_MILLIS, scheduler); + try { + // Must not throw if record throws: + eventProcessor.record(new Event(System.currentTimeMillis(), CONTEXT) { + @Override + public long getSamplingRatio() { + throw new RuntimeException("simulated record crash"); + } + }); + logging.assertErrorLogged("Unexpected error in event processor: java.lang.RuntimeException: simulated record crash"); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void anErrorWhileRecordingStillReachesTheCaller() throws Exception { + // Only exceptions are the SDK's to absorb. An Error such as OutOfMemoryError belongs to the + // application's crash reporting, and swallowing it on the caller's thread would hide it. + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + NO_PERIODIC_FLUSH_MILLIS, scheduler); + try { + eventProcessor.record(new Event(System.currentTimeMillis(), CONTEXT) { + @Override + public long getSamplingRatio() { + throw new StackOverflowError("simulated"); + } + }); + fail("the Error was swallowed"); + } catch (StackOverflowError expected) { + // what the application's handler would see + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + @Test + public void aFailedCommitOnTheCallersThreadDoesNotReachTheCaller() throws Exception { + // With IMMEDIATE the commit runs inside track, identify and flush, after record() has + // returned, so it needs its own guard. + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", DEFAULT_CAPACITY, true, + logging.logger) { + @Override + void commit() { + throw new IllegalStateException("simulated commit failure"); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + UNUSED_EVENTS_URI, null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY, true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + eventProcessor.recordIdentifyEvent(CONTEXT); + eventProcessor.flush(); + // Its own last-chance commit runs on the caller's thread when the events thread's fails. + eventProcessor.close(); + + logging.assertErrorLogged( + "Unexpected error in event processor: java.lang.IllegalStateException: simulated commit failure"); + } finally { + scheduler.shutdownNow(); + } + } + + @Test + public void closeGivesUpWaitingOnAStalledDelivery() throws Exception { + // close() runs on the caller's thread, usually the main one, so a send that never comes back + // used to park the application there for as long as the HTTP timeouts allowed. + Semaphore letResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.all(Handlers.waitFor(letResponseFinish), + Handlers.status(202)))) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler); + try { + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "stalled", LDValue.ofNull(), null); + + long startedAtNanos = System.nanoTime(); + eventProcessor.close(); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAtNanos); + + assertNotNull("the sender was never asked to send anything", + server.getRecorder().requireRequest(2, TimeUnit.SECONDS)); + assertTrue("close() waited " + elapsedMillis + "ms on a budget of " + + CLOSE_BUDGET_MILLIS + "ms", elapsedMillis < CLOSE_BUDGET_MILLIS * 5); + logging.assertWarnLogged("Gave up waiting for the final event delivery"); + } finally { + letResponseFinish.release(Integer.MAX_VALUE); + scheduler.shutdownNow(); + } + } + } + + @Test + public void closeGivesUpWaitingOnAStalledDiskWrite() throws Exception { + // A commit stalled on the disk holds commitLock, and the store's I/O lock, for as long as the + // write takes. close() used to fall back to committing on the caller's thread once its budget + // ran out, which then waited on those locks with no bound at all. + CountDownLatch writeStarted = new CountDownLatch(1); + Semaphore letWriteFinish = new Semaphore(0); + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", DEFAULT_CAPACITY, true, + logging.logger) { + @Override + FileOutputStream openForAppending(File log) throws IOException { + writeStarted.countDown(); + awaitQuietly(letWriteFinish); + return super.openForAppending(log); + } + }; + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + UNUSED_EVENTS_URI, null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY, false); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try { + // A commit point, so a commit is queued and stalls on the disk, holding the locks. + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + assertTrue("the commit never reached the disk", writeStarted.await(5, TimeUnit.SECONDS)); + + long startedAtNanos = System.nanoTime(); + // On a thread of its own, so that a close() that does hang fails this test rather than + // hanging the suite. + Future closing = caller.submit(() -> { + eventProcessor.close(); + return null; + }); + closing.get(CLOSE_BUDGET_MILLIS * 5, TimeUnit.MILLISECONDS); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAtNanos); + + assertTrue("close() waited " + elapsedMillis + "ms on a budget of " + + CLOSE_BUDGET_MILLIS + "ms", elapsedMillis < CLOSE_BUDGET_MILLIS * 3); + logging.assertWarnLogged("Gave up waiting for the final event commit"); + } finally { + letWriteFinish.release(Integer.MAX_VALUE); + caller.shutdownNow(); + scheduler.shutdownNow(); + } + } + + private static void awaitQuietly(Semaphore semaphore) { + try { + if (semaphore.tryAcquire(30, TimeUnit.SECONDS)) { + semaphore.release(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @Test + public void flushWithTimeoutReportsFailureWhileOffline() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.setOffline(true); + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + // The events are still buffered rather than delivered, and no amount of waiting + // changes that, so the caller is told so instead of being told they are safe. + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void flushesArrivingWhileADeliveryRunsShareOneFollowUpDelivery() throws Exception { + // Otherwise a flush called faster than a post completes queues a post per call, and the + // flush that matters -- the one at shutdown, with a deadline -- waits behind all of them. + Semaphore letFirstResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.sequential( + Handlers.all(Handlers.waitFor(letFirstResponseFinish), Handlers.status(202)), + Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "first", LDValue.ofNull(), null); + Future first = eventProcessor.flushAsync(); + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + + // The delivery thread is inside that post, so none of these can start, and each of + // them has to be answered by the one delivery that is queued behind it. + eventProcessor.recordCustomEvent(CONTEXT, "second", LDValue.ofNull(), null); + Future queued = eventProcessor.flushAsync(); + for (int i = 0; i < 50; i++) { + assertSame(queued, eventProcessor.flushAsync()); + } + + letFirstResponseFinish.release(1); + assertTrue(first.get(5, TimeUnit.SECONDS)); + assertTrue(queued.get(5, TimeUnit.SECONDS)); + + // One post for the running delivery and one for the 51 that joined, and no more. + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + } finally { + letFirstResponseFinish.release(Integer.MAX_VALUE); + eventProcessor.close(); + } + } + } + + @Test + public void aFlushJoiningADeliveryStillCoversWhatTheCallerRecorded() throws Exception { + // Joining is only sound while the delivery it joins has not taken the buffer yet, so what + // the joining caller recorded has to come back in that delivery's payload. + Semaphore letFirstResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.sequential( + Handlers.all(Handlers.waitFor(letFirstResponseFinish), Handlers.status(202)), + Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "first", LDValue.ofNull(), null); + Future first = eventProcessor.flushAsync(); + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + + eventProcessor.recordCustomEvent(CONTEXT, "joined", LDValue.ofNull(), null); + Future queued = eventProcessor.flushAsync(); + + letFirstResponseFinish.release(1); + assertTrue(first.get(5, TimeUnit.SECONDS)); + assertTrue(queued.get(5, TimeUnit.SECONDS)); + + RequestInfo second = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue("the joining caller's event was left behind", + second.getBody().contains("\"key\":\"joined\"")); + } finally { + letFirstResponseFinish.release(Integer.MAX_VALUE); + eventProcessor.close(); + } + } + } + + @Test + public void flushWithTimeoutReportsFailureWhenTheTimeoutExpiresFirst() throws Exception { + Semaphore letResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.all(Handlers.waitFor(letResponseFinish), + Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + Future delivery = eventProcessor.flushAsync(); + + try { + delivery.get(100, TimeUnit.MILLISECONDS); + fail("the delivery finished while its response was still being held"); + } catch (TimeoutException expected) { + // The caller gives up here, as flushAndWait does when its budget runs out. + } + + // Left running rather than canceled, the delivery still gets its events through + // once the service answers, and a canceled one could not report that. + letResponseFinish.release(1); + assertTrue(delivery.get(5, TimeUnit.SECONDS)); + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + } finally { + // Released before closing, so that the delivery still in flight can finish rather + // than hold up the shutdown that close() waits on. + letResponseFinish.drainPermits(); + letResponseFinish.release(Integer.MAX_VALUE); + eventProcessor.close(); + } + } + } + + @Test + public void flushReportsFailureWhenTheServiceRefusesTheEventsForNow() throws Exception { + // 503 is a failure that may pass: the processor retries it once, and a caller waiting on the + // flush is answered by that retry. The batch is kept after it; see the tests below. + try (HttpServer server = HttpServer.start(Handlers.status(503))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void flushReportsFailureWhenTheServiceRefusesTheEventsForGood() throws Exception { + // 401 is not retried, and stops the processor for the life of the process, so the flush + // after it has nothing it can deliver on either. + try (HttpServer server = HttpServer.start(Handlers.status(401))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "refused", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + + eventProcessor.recordCustomEvent(CONTEXT, "after", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void flushAndWaitDoesNotReportDeliveryForEventsAnEarlierFailedFlushDrained() throws Exception { + // Every request fails recoverably (503): each delivery attempts, retries once, and keeps the batch. + try (HttpServer server = HttpServer.start(Handlers.status(503))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "kept-event", LDValue.ofNull(), null); + + // Stands in for the periodic flush or the SDK's flush-on-background: it closes the + // buffer into a batch and its delivery fails. The batch stays in the store. + eventProcessor.blockingFlush(); + server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + + // The buffer is empty, but the event is still waiting in its batch, not delivered. + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aBatchAnUnheardDeliveryCouldNotSendGoesOutOnceTheServiceRecovers() throws Exception { + HandlerSwitcher service = new HandlerSwitcher(Handlers.status(503)); + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "kept-event", LDValue.ofNull(), null); + eventProcessor.blockingFlush(); + // The attempt and its one retry: the service only recovers once both have failed. + String payloadId = server.getRecorder().requireRequest(10, TimeUnit.SECONDS) + .getHeader(PAYLOAD_ID_HEADER); + server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + service.setTarget(Handlers.status(202)); + + // Nobody heard the failure, and the next flush still sends what it refused. + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + RequestInfo resent = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + assertTrue(resent.getBody().contains("\"key\":\"kept-event\"")); + assertEquals("the same delivery, so the service can discard a repeat", + payloadId, resent.getHeader(PAYLOAD_ID_HEADER)); + + // Delivered once: the next flush has only its own events to send. + eventProcessor.recordCustomEvent(CONTEXT, "later-event", LDValue.ofNull(), null); + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + RequestInfo later = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + assertTrue(later.getBody().contains("\"key\":\"later-event\"")); + assertFalse(later.getBody().contains("kept-event")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aBatchIsKeptThroughRepeatedRefusalsAndDeliveredOnceTheServiceRecovers() throws Exception { + assertKeptThroughRefusalsAndDeliveredOnRecovery(503, eventsBuilder(DEFAULT_CAPACITY), 3); + } + + @Test + public void aBatchHeldInMemoryIsKeptThroughRepeatedRefusalsWherePersistenceIsOff() throws Exception { + // Without a disk the store holds batches in memory, and a refusal must not cost them either. + assertKeptThroughRefusalsAndDeliveredOnRecovery(503, + eventsBuilder(DEFAULT_CAPACITY).eventPersistence(EventPersistence.DISABLED), 3); + } + + @Test + public void everyRecoverableStatusKeepsTheBatchPastTheRetry() throws Exception { + // The statuses the SDK treats as worth trying again; each has to keep the batch, not just 503. + for (int status : new int[] { 400, 408, 429, 500, 502, 504 }) { + assertKeptThroughRefusalsAndDeliveredOnRecovery(status, eventsBuilder(DEFAULT_CAPACITY), 2); + } + } + + @Test + public void eventsRecordedDuringAnOutageAreDeliveredWithTheBatchesRefusedBeforeThem() throws Exception { + HandlerSwitcher service = new HandlerSwitcher(Handlers.status(503)); + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "first-event", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + eventProcessor.recordCustomEvent(CONTEXT, "second-event", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + // Two flushes, each an attempt and a retry, all at the first batch: delivery stops at + // the oldest batch that fails in a way that may pass. + for (int i = 0; i < 4; i++) { + assertTrue(server.getRecorder().requireRequest(5, TimeUnit.SECONDS).getBody() + .contains("\"key\":\"first-event\"")); + } + service.setTarget(Handlers.status(202)); + + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + // Oldest first, each exactly once. + RequestInfo first = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + RequestInfo second = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + assertTrue(first.getBody().contains("\"key\":\"first-event\"")); + assertFalse(first.getBody().contains("second-event")); + assertTrue(second.getBody().contains("\"key\":\"second-event\"")); + assertFalse(second.getBody().contains("first-event")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aBatchStillRefusedAtCloseIsDeliveredByTheNextRun() throws Exception { + assumePersisting(); + String payloadId; + try (HttpServer refusing = HttpServer.start(Handlers.status(503))) { + EventProcessor eventProcessor = makeEventProcessor(refusing, DEFAULT_CAPACITY); + eventProcessor.recordCustomEvent(CONTEXT, "kept-event", LDValue.ofNull(), null); + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + payloadId = refusing.getRecorder().requireRequest(1, TimeUnit.SECONDS) + .getHeader(PAYLOAD_ID_HEADER); + eventProcessor.close(); + } + + // The same directory, as the application's next launch would find it. + try (HttpServer server = startEventsServer()) { + EventProcessor nextRun = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + assertTrue(awaitFlush(nextRun, 10, TimeUnit.SECONDS)); + RequestInfo resent = server.getRecorder().requireRequest(10, TimeUnit.SECONDS); + assertTrue(resent.getBody().contains("\"key\":\"kept-event\"")); + assertEquals(payloadId, resent.getHeader(PAYLOAD_ID_HEADER)); + } finally { + nextRun.close(); + } + } + } + + @Test + public void aBatchRefusedWith400IsPostedOnlyByEachFlushAndItsRetry() throws Exception { + assertPostedOnlyByEachFlushAndItsRetry(400); + } + + @Test + public void aBatchRefusedWith503IsPostedOnlyByEachFlushAndItsRetry() throws Exception { + assertPostedOnlyByEachFlushAndItsRetry(503); + } + + /** + * Kept is not the same as resent on its own: a failure must never queue another attempt beyond the + * one retry, or a batch the service always refuses would be posted in a loop. + */ + private void assertPostedOnlyByEachFlushAndItsRetry(int status) throws Exception { + try (HttpServer server = HttpServer.start(Handlers.status(status))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "refused", LDValue.ofNull(), null); + for (int flush = 0; flush < 2; flush++) { + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + server.getRecorder().requireNoRequests(DirectEventProcessor.RETRY_DELAY_MILLIS * 3, + TimeUnit.MILLISECONDS); + } + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void flushesDuringARefusedDeliveryShareOneDeliveryAndOneRetry() throws Exception { + Semaphore letFirstResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.all(Handlers.waitFor(letFirstResponseFinish), + Handlers.status(503)))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "refused", LDValue.ofNull(), null); + List> flushes = new ArrayList<>(); + flushes.add(eventProcessor.flushAsync()); + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + + // Held on the first post, every one of these joins the single delivery queued behind it. + for (int i = 0; i < 20; i++) { + flushes.add(eventProcessor.flushAsync()); + } + letFirstResponseFinish.release(100); + + for (Future flush : flushes) { + assertFalse(flush.get(10, TimeUnit.SECONDS)); + } + // The second delivery, then the one retry both failures share. + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + server.getRecorder().requireRequest(1, TimeUnit.SECONDS); + server.getRecorder().requireNoRequests(DirectEventProcessor.RETRY_DELAY_MILLIS * 3, + TimeUnit.MILLISECONDS); + } finally { + letFirstResponseFinish.release(100); + eventProcessor.close(); + } + } + } + + @Test + public void thePeriodicFlushPostsARefusedBatchAtMostOncePerInterval() throws Exception { + int intervalMillis = 200; + long windowMillis = 2_000; + try (HttpServer server = HttpServer.start(Handlers.status(503))) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).flushIntervalMillis(intervalMillis), true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "refused", LDValue.ofNull(), null); + Thread.sleep(windowMillis); + + // One post per interval and one outstanding retry at a time; a loop would be in the + // hundreds by now. + int posts = server.getRecorder().count(); + long most = windowMillis / intervalMillis + 1 + + windowMillis / DirectEventProcessor.RETRY_DELAY_MILLIS + 1; + assertTrue(posts + " posts in " + windowMillis + "ms", posts >= 3 && posts <= most); + + // Offline stops it entirely, kept batch or not. + eventProcessor.setOffline(true); + Thread.sleep(DirectEventProcessor.RETRY_DELAY_MILLIS + intervalMillis); + int whenOffline = server.getRecorder().count(); + Thread.sleep(intervalMillis * 3); + assertEquals(whenOffline, server.getRecorder().count()); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aBatchAPreviousRunLeftIsTriedOnceAtStartAndThenWaitsWhileStillRefused() throws Exception { + assumePersisting(); + try (HttpServer refusing = HttpServer.start(Handlers.status(503))) { + EventProcessor eventProcessor = makeEventProcessor(refusing, DEFAULT_CAPACITY); + eventProcessor.recordCustomEvent(CONTEXT, "kept-event", LDValue.ofNull(), null); + eventProcessor.close(); + } + + try (HttpServer server = HttpServer.start(Handlers.status(503))) { + EventProcessor nextRun = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + // The delivery at start and its retry, and nothing after them until a flush asks. + assertTrue(server.getRecorder().requireRequest(5, TimeUnit.SECONDS).getBody() + .contains("\"key\":\"kept-event\"")); + server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + server.getRecorder().requireNoRequests(DirectEventProcessor.RETRY_DELAY_MILLIS * 3, + TimeUnit.MILLISECONDS); + } finally { + nextRun.close(); + } + } + } + + /** + * Refuses a batch with {@code status} for several flushes in a row, each of which attempts it and + * retries it once, then lets the service recover and checks that the batch arrives exactly once, + * under the payload ID it was first sent with. + */ + private void assertKeptThroughRefusalsAndDeliveredOnRecovery(int status, EventProcessorBuilder events, + int refusedFlushes) throws Exception { + HandlerSwitcher service = new HandlerSwitcher(Handlers.status(status)); + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = makeEventProcessor(server, events, true); + try { + eventProcessor.recordCustomEvent(CONTEXT, "kept-event", LDValue.ofNull(), null); + String payloadId = null; + for (int flush = 0; flush < refusedFlushes; flush++) { + assertFalse("HTTP " + status + ", flush " + flush, + awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + for (int attempt = 0; attempt < 2; attempt++) { + RequestInfo refused = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue("HTTP " + status + ": the batch was dropped before flush " + flush, + refused.getBody().contains("\"key\":\"kept-event\"")); + if (payloadId == null) { + payloadId = refused.getHeader(PAYLOAD_ID_HEADER); + } + assertEquals(payloadId, refused.getHeader(PAYLOAD_ID_HEADER)); + } + } + service.setTarget(Handlers.status(202)); + + assertTrue("HTTP " + status + ": not delivered once the service recovered", + awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + RequestInfo delivered = server.getRecorder().requireRequest(5, TimeUnit.SECONDS); + assertTrue(delivered.getBody().contains("\"key\":\"kept-event\"")); + assertEquals(payloadId, delivered.getHeader(PAYLOAD_ID_HEADER)); + + // Gone once accepted: nothing is left to resend. + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aFlushIsNotToldEventsArrivedThatCouldNotBeSerialized() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "poison", LDValue.ofNull(), Double.NaN); + + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aFlushIsNotToldEventsArrivedWhenSomeOfThemCouldNotBeSerialized() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "poison", LDValue.ofNull(), Double.NaN); + eventProcessor.recordCustomEvent(CONTEXT, "fine", LDValue.ofNull(), 1.0); + + // The post succeeds, and still not everything the caller recorded is in it. + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + List events = collectDelivered(server); + assertEquals(LDValue.of("fine"), requireEventOfKind(events, "custom").get("key")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aFlushIsNotToldEventsArrivedWhenOneWasTooLargeToStore() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + char[] data = new char[EventLogFormat.MAX_FRAME_SIZE]; + Arrays.fill(data, 'x'); + eventProcessor.recordCustomEvent(CONTEXT, "too-large", LDValue.of(new String(data)), null); + eventProcessor.recordCustomEvent(CONTEXT, "fine", LDValue.ofNull(), null); + + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + List events = collectDelivered(server); + assertEquals(1, countEventsOfKind(events, "custom")); + assertEquals(LDValue.of("fine"), requireEventOfKind(events, "custom").get("key")); + logging.assertErrorLogged("too large to store"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aLossAfterAFlushClosedItsBatchIsReportedToTheNextFlushInstead() throws Exception { + assumePersisting(); + // A flush answers for the batch it closed. One that met a retryable failure answers only when + // its retry does, which is long enough for a loss to happen in between, and that loss belongs + // to the flush that closes the batch after it rather than to this one. + char[] data = new char[EventLogFormat.MAX_FRAME_SIZE]; + Arrays.fill(data, 'x'); + LDValue tooLarge = LDValue.of(new String(data)); + Semaphore letFirstResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.sequential( + Handlers.all(Handlers.waitFor(letFirstResponseFinish), Handlers.status(503)), + Handlers.status(202)))) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + Future firstFlush = eventProcessor.flushAsync(); + // The batch is closed and on its way by the time the request lands, so the loss below + // happens after this flush took everything it can speak for. + assertNotNull(server.getRecorder().requireRequest(5, TimeUnit.SECONDS)); + + eventProcessor.recordCustomEvent(CONTEXT, "too-large", tooLarge, null); + logging.assertErrorLogged("too large to store"); + // One permit, not every permit: only this first response is held, and the finally + // below has to be able to release it again without overflowing the count. + letFirstResponseFinish.release(); + + // The retry sends the batch this flush closed, and everything in it arrives. + assertTrue("told about a loss that happened after it closed its batch", + Boolean.TRUE.equals(firstFlush.get(10, TimeUnit.SECONDS))); + // And the loss is still owed to somebody, so the next caller is the one to hear it. + assertFalse(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + } finally { + letFirstResponseFinish.release(); + eventProcessor.close(); + } + } + } + + @Test + public void eventsHeldInMemoryWhenAWriteFailsAreDeliveredWithThoseWrittenBeforeIt() throws Exception { + AtomicInteger writes = new AtomicInteger(); + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", DEFAULT_CAPACITY, true, + logging.logger) { + @Override + FileOutputStream openForAppending(File log) throws IOException { + return new FileOutputStream(log, true) { + @Override + public void write(byte[] bytes) throws IOException { + // The file header and the first event land; the disk is full from then on. + if (writes.incrementAndGet() > 2) { + throw new IOException("No space left on device"); + } + super.write(bytes); + } + }; + } + }; + try (HttpServer server = startEventsServer()) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY, true); + try { + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "written", LDValue.ofNull(), null); + // Its write fails, so persistence is given up with "written" in the log and this in memory. + eventProcessor.recordCustomEvent(CONTEXT, "held", LDValue.ofNull(), null); + logging.assertWarnLogged("Giving up on persisting events"); + + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + + List keys = new ArrayList<>(); + for (LDValue event : collectDelivered(server)) { + keys.add(event.get("key").stringValue()); + } + assertEquals("oldest first, and neither left behind", + Arrays.asList("written", "held"), keys); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + } + + @Test + public void aBatchThatCannotBeReadIsKeptAndDeliveredOnceItCanBe() throws Exception { + AtomicBoolean failing = new AtomicBoolean(false); + EventStore store = new EventStore(eventsDirectory.newFolder(), "test", DEFAULT_CAPACITY, true, + logging.logger) { + @Override + InputStream openForReading(File file) throws IOException { + if (failing.get()) { + // What a process out of file descriptors gets for a file that is there. + throw new FileNotFoundException(file + " (Too many open files)"); + } + return super.openForReading(file); + } + }; + try (HttpServer server = startEventsServer()) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, DEFAULT_CAPACITY, true); + try { + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "kept", LDValue.ofNull(), null); + + failing.set(true); + assertFalse("a batch that could not be read was reported as delivered", + awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + server.getRecorder().requireNoRequests(100, TimeUnit.MILLISECONDS); + logging.assertWarnLogged("Could not read stored events"); + + failing.set(false); + assertTrue(awaitFlush(eventProcessor, 10, TimeUnit.SECONDS)); + List events = collectDelivered(server); + assertEquals(1, countEventsOfKind(events, "custom")); + assertEquals(LDValue.of("kept"), requireEventOfKind(events, "custom").get("key")); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + } + + @Test + public void closeReleasesTheStoreAndSenderOnlyAfterTheLastDeliveryFinishes() throws Exception { + // Giving up on the wait must not turn into pulling the store or the HTTP client out from + // under the delivery we just decided not to wait for. Neither can be observed being closed + // from here, but the consequence can: a delivery that lost either would fail once released + // and leave its batch on disk instead of deleting it. + Semaphore letResponseFinish = new Semaphore(0); + try (HttpServer server = HttpServer.start(Handlers.all(Handlers.waitFor(letResponseFinish), + Handlers.status(202)))) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(new StubEventSender(), + server.getUri(), null, NO_PERIODIC_FLUSH_MILLIS, 60_000, CLOSE_BUDGET_MILLIS, + scheduler); + try { + eventProcessor.setOffline(false); + eventProcessor.recordCustomEvent(CONTEXT, "stalled", LDValue.ofNull(), null); + + eventProcessor.close(); // returns on its budget with the post still stalled + letResponseFinish.release(Integer.MAX_VALUE); + + assertTrue("the stalled delivery never completed, so its batch is still on disk", + awaitNoPendingEvents(5, TimeUnit.SECONDS)); + } finally { + // Drained first, because the release above may already have happened and topping a + // semaphore up twice from Integer.MAX_VALUE overflows its permit count. + letResponseFinish.drainPermits(); + letResponseFinish.release(Integer.MAX_VALUE); + scheduler.shutdownNow(); + } + } + } + + private boolean awaitNoPendingEvents(long timeout, TimeUnit unit) throws Exception { + long deadlineNanos = System.nanoTime() + unit.toNanos(timeout); + do { + EventStore reader = EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, "test", + DEFAULT_CAPACITY, true, logging.logger); + try { + if (reader.pendingEventPayloads().isEmpty()) { + return true; + } + } finally { + reader.close(); + } + Thread.sleep(25); + } while (System.nanoTime() < deadlineNanos); + return false; + } + + @Test + public void flushAfterCloseDoesNotDeliverThroughReleasedResources() throws Exception { + // close() hands the store and the analytics sender back on the delivery thread and then stops + // that thread taking work, so nothing it accepts afterwards can find them already released. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + eventProcessor.recordCustomEvent(CONTEXT, "before-close", LDValue.ofNull(), null); + // Leaves the buffer empty, so any request after this is one close should not have made. + flushAndCollect(eventProcessor, server); + + eventProcessor.close(); + + eventProcessor.recordCustomEvent(CONTEXT, "after-close", LDValue.ofNull(), null); + eventProcessor.flush(); + eventProcessor.blockingFlush(); + + server.getRecorder().requireNoRequests(200, TimeUnit.MILLISECONDS); + } + } + + @Test + public void stalledDiagnosticPostDoesNotHoldUpAnalyticsDelivery() throws Exception { + // Diagnostics used to share the one thread that delivers analytics, so a post against a + // network that accepts connections and never answers stalled every flush behind it, and a + // buffer that is not being drained fills up and drops what the application asked to send. + CountDownLatch diagnosticStarted = new CountDownLatch(1); + CountDownLatch releaseDiagnostic = new CountDownLatch(1); + EventSender sender = new StubEventSender() { + @Override + public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) { + diagnosticStarted.countDown(); + awaitQuietly(releaseDiagnostic, 5, TimeUnit.SECONDS); + return new Result(true, false, null); + } + }; + + // Analytics go out through AnalyticsEventSender rather than the injectable one, so the + // delivery has to be watched at the server rather than at the stub. + try (HttpServer server = startEventsServer()) { + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + DirectEventProcessor eventProcessor = makeEventProcessor(sender, server.getUri(), + makeDiagnosticStore(), NO_PERIODIC_FLUSH_MILLIS, 60_000, + DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, scheduler); + try { + // Coming online posts the diagnostic init event, which then never comes back. + eventProcessor.setOffline(false); + assertTrue("the diagnostic event was never posted", + diagnosticStarted.await(2, TimeUnit.SECONDS)); + + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + eventProcessor.flush(); + + RequestInfo request = null; + try { + request = server.getRecorder().requireRequest(2, TimeUnit.SECONDS); + } catch (Exception timedOut) { + // Reported by the assertion below, which can say what it means. + } + assertNotNull("analytics delivery was stuck behind the diagnostic post", request); + assertTrue(request.getBody().contains("an-event")); + } finally { + releaseDiagnostic.countDown(); + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + } + + @Test + public void diagnosticEventIsDroppedRatherThanQueuedBehindOneStillPosting() throws Exception { + CountDownLatch firstStarted = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + AtomicInteger posts = new AtomicInteger(); + EventSender sender = new StubEventSender() { + @Override + public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) { + posts.incrementAndGet(); + firstStarted.countDown(); + awaitQuietly(releaseFirst, 5, TimeUnit.SECONDS); + return new Result(true, false, null); + } + }; + + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + // A diagnostic interval short enough that the periodic task fires repeatedly while the init + // event is still stuck on the posting thread. + DirectEventProcessor eventProcessor = makeEventProcessor(sender, makeDiagnosticStore(), + NO_PERIODIC_FLUSH_MILLIS, 20, scheduler); + try { + eventProcessor.setOffline(false); + assertTrue("the diagnostic event was never posted", + firstStarted.await(2, TimeUnit.SECONDS)); + + // Long enough for many periodic runs, every one of which has to be turned away rather + // than left on the posting thread's queue. Counting posts is not enough on its own: a + // queued one would not have started yet either, so the skips are what distinguishes + // being dropped from merely waiting. + Thread.sleep(300); + assertTrue("no diagnostic event was turned away, so they were queueing up instead", + countLogged("Skipped a diagnostic event") > 0); + assertEquals("more than one diagnostic event reached the sender", 1, posts.get()); + } finally { + releaseFirst.countDown(); + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + private static final URI UNUSED_EVENTS_URI = URI.create("https://events.example"); + + @Test + public void goingToTheBackgroundDefersADiagnosticPeriodRatherThanDestroyingIt() throws Exception { + // createEventAndReset hands the period's statistics back and clears them in the same call, so + // whoever consumes it owns it. The post reaches the diagnostics thread through a queue it may + // have to wait in, and the SDK can go offline or to the background in the meantime. If that is + // noticed after the event was built, the period is gone: nothing retries it, and the next + // event reports a window starting after the reset, so the statistics are not merely late but + // absent. + BlockingQueue posted = new LinkedBlockingQueue<>(); + EventSender sender = new StubEventSender() { + @Override + public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) { + posted.add(LDValue.parse(new String(data, StandardCharsets.UTF_8))); + return new Result(true, false, null); + } + }; + + ScheduledExecutorService scheduler = EventUtil.makeEventsTaskExecutor(); + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + // Short enough that the periodic task fires while the diagnostics thread is held. + DirectEventProcessor eventProcessor = makeEventProcessor(sender, UNUSED_EVENTS_URI, + makeDiagnosticStore(), NO_PERIODIC_FLUSH_MILLIS, 20, + DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, scheduler, diagnosticExecutor); + try { + eventProcessor.setOffline(false); + assertEquals(LDValue.of("diagnostic-init"), + requirePosted(posted).get("kind")); + + // Something for the period to have in it. Capacity is what makes these countable, and the + // dropped count is carried by the periodic event and nothing else. Far past capacity so + // that drops happen whatever else is draining the buffer. + for (int i = 0; i < DEFAULT_CAPACITY * 10; i++) { + eventProcessor.recordCustomEvent(CONTEXT, "an-event", LDValue.ofNull(), null); + } + logging.assertWarnLogged("Exceeded event queue capacity"); + + // Holds the diagnostics thread so the next periodic post has to queue behind it, which is + // the window the state can change in. + CountDownLatch releaseThread = new CountDownLatch(1); + diagnosticExecutor.submit(() -> awaitQuietly(releaseThread, 5, TimeUnit.SECONDS)); + // A later run being turned away is proof that an earlier one took the claim and is sitting + // on the queue, which is what we need before changing the state underneath it. + awaitLogged("Skipped a diagnostic event"); + + eventProcessor.setInBackground(true); + releaseThread.countDown(); + // The queued post now runs suspended. Nothing should reach the sender. + assertNull("a diagnostic event was posted from the background", + posted.poll(300, TimeUnit.MILLISECONDS)); + + eventProcessor.setInBackground(false); + LDValue statistics = requirePosted(posted); + + assertEquals(LDValue.of("diagnostic"), statistics.get("kind")); + // Nothing is recorded after the suspension, so every drop this could report happened + // before it. A destroyed period therefore reports exactly zero, which is what separates + // the two outcomes. The exact figure is not asserted because it is not the same on every + // tier: where events are staged to a store, reaching it frees capacity as we go. + assertTrue("the suspended period was destroyed rather than carried forward", + statistics.get("droppedEvents").longValue() > 0); + } finally { + eventProcessor.close(); + scheduler.shutdownNow(); + } + } + + private LDValue requirePosted(BlockingQueue posted) throws InterruptedException { + LDValue event = posted.poll(5, TimeUnit.SECONDS); + assertNotNull("no diagnostic event was posted", event); + return event; + } + + /** + * Goes to the background and back, leaving the processor at the point where it decides whether + * to send a second init event. + *

+ * Two things have to be true for that decision to be reached, and only one of them is under the + * test's control. Going to the foreground has to be a real transition, because setInBackground + * returns immediately when the value is unchanged. And the diagnostics thread has to be free: + * the first init event holds a claim on it until its post returns, which is after the request + * reaches the server, so a transition right after the request arrives is turned away before it + * gets anywhere near the decision. Being turned away is logged, which is what this waits out. + */ + private void returnToTheForegroundWithTheDiagnosticsThreadFree(EventProcessor eventProcessor) + throws InterruptedException { + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + long skipsBefore = countLogged("Skipped a diagnostic event"); + eventProcessor.setInBackground(true); + eventProcessor.setInBackground(false); + // Logged by the claim itself, on this thread, so it is already there if it happened. + if (countLogged("Skipped a diagnostic event") == skipsBefore) { + return; + } + Thread.sleep(10); + } + throw new AssertionError("the diagnostics thread never came free"); + } + + private void awaitLogged(String messageSubstring) throws InterruptedException { + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + if (countLogged(messageSubstring) > 0) { + return; + } + Thread.sleep(10); + } + throw new AssertionError("never logged: " + messageSubstring); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + long flushIntervalMillis, + ScheduledExecutorService scheduler) { + return makeEventProcessor(diagnosticSender, UNUSED_EVENTS_URI, null, flushIntervalMillis, + 60_000, DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, scheduler); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + DiagnosticStore diagnosticStore, + long flushIntervalMillis, + ScheduledExecutorService scheduler) { + return makeEventProcessor(diagnosticSender, UNUSED_EVENTS_URI, diagnosticStore, flushIntervalMillis, + 60_000, DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, scheduler); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + DiagnosticStore diagnosticStore, + long flushIntervalMillis, + long diagnosticIntervalMillis, + ScheduledExecutorService scheduler) { + return makeEventProcessor(diagnosticSender, UNUSED_EVENTS_URI, diagnosticStore, flushIntervalMillis, + diagnosticIntervalMillis, DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, + scheduler); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + URI eventsUri, + DiagnosticStore diagnosticStore, + long flushIntervalMillis, + long diagnosticIntervalMillis, + long closeBudgetMillis, + ScheduledExecutorService scheduler) { + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + return makeEventProcessor(diagnosticSender, eventsUri, diagnosticStore, flushIntervalMillis, + diagnosticIntervalMillis, closeBudgetMillis, scheduler, diagnosticExecutor); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + URI eventsUri, + DiagnosticStore diagnosticStore, + long flushIntervalMillis, + long diagnosticIntervalMillis, + long closeBudgetMillis, + ScheduledExecutorService scheduler, + ExecutorService diagnosticExecutor) { + return makeEventProcessor(diagnosticSender, eventsUri, diagnosticStore, flushIntervalMillis, + diagnosticIntervalMillis, closeBudgetMillis, scheduler, diagnosticExecutor, + EventStore.create(eventsDirectory.getRoot(), MOBILE_KEY, "test", DEFAULT_CAPACITY, + true, logging.logger), + DEFAULT_CAPACITY); + } + + private DirectEventProcessor makeEventProcessor(EventStore store, int capacity, + ScheduledExecutorService scheduler) { + ExecutorService diagnosticExecutor = EventUtil.makeDiagnosticsTaskExecutor(); + diagnosticExecutors.add(diagnosticExecutor); + return makeEventProcessor(new StubEventSender(), UNUSED_EVENTS_URI, null, + NO_PERIODIC_FLUSH_MILLIS, 60_000, DirectEventProcessor.DEFAULT_CLOSE_BUDGET_MILLIS, + scheduler, diagnosticExecutor, store, capacity); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + URI eventsUri, + DiagnosticStore diagnosticStore, + long flushIntervalMillis, + long diagnosticIntervalMillis, + long closeBudgetMillis, + ScheduledExecutorService scheduler, + ExecutorService diagnosticExecutor, + EventStore store, + int capacity) { + return makeEventProcessor(diagnosticSender, eventsUri, diagnosticStore, flushIntervalMillis, + diagnosticIntervalMillis, closeBudgetMillis, scheduler, diagnosticExecutor, store, + capacity, false); + } + + private DirectEventProcessor makeEventProcessor(EventSender diagnosticSender, + URI eventsUri, + DiagnosticStore diagnosticStore, + long flushIntervalMillis, + long diagnosticIntervalMillis, + long closeBudgetMillis, + ScheduledExecutorService scheduler, + ExecutorService diagnosticExecutor, + EventStore store, + int capacity, + boolean commitOnCallerThread) { + ExecutorService commitExecutor = EventUtil.makeEventsCommitExecutor(); + diagnosticExecutors.add(commitExecutor); + lastCommitExecutor = commitExecutor; + return new DirectEventProcessor( + new OutboundEventBuffer(false, Collections.emptyList(), true, capacity, + logging.logger), + store, + diagnosticSender, + new AnalyticsEventSender(LDUtil.makeHttpProperties( + new HttpConfiguration(2000, Collections.emptyMap(), null, false)), + logging.logger), + eventsUri, + diagnosticStore, + capacity, + commitOnCallerThread, + flushIntervalMillis, + diagnosticIntervalMillis, + closeBudgetMillis, + false, + true, // initiallyOffline, as the SDK builds it + scheduler, + commitExecutor, + diagnosticExecutor, + logging.logger); + } + + private long countLogged(String messageSubstring) { + long count = 0; + for (String message : logging.logCapture.getMessageStrings()) { + if (message.contains(messageSubstring)) { + count++; + } + } + return count; + } + + private DiagnosticStore makeDiagnosticStore() { + return new DiagnosticStore(new DiagnosticStore.SdkDiagnosticParams(MOBILE_KEY, + "android-client-sdk", "0.0.0", "Android", null, Collections.emptyMap(), null)); + } + + /** + * Flushes and waits for the outcome the way {@code LDClient.flushAndWait} does, which is the + * only place a timeout belongs. + */ + private static boolean awaitFlush(EventProcessor eventProcessor, long timeout, TimeUnit unit) + throws Exception { + try { + return Boolean.TRUE.equals(eventProcessor.flushAsync().get(timeout, unit)); + } catch (TimeoutException e) { + return false; + } + } + + private static void awaitQuietly(CountDownLatch latch, long timeout, TimeUnit unit) { + try { + latch.await(timeout, unit); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** Accepts everything, so that a test overrides only the one method it is about. */ + private static class StubEventSender implements EventSender { + @Override + public Result sendAnalyticsEvents(byte[] data, int eventCount, URI eventsBaseUri) { + return new Result(true, false, null); + } + + @Override + public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) { + return new Result(true, false, null); + } + + @Override + public void close() {} + } + + private void recordEvaluation(EventProcessor eventProcessor, boolean requireFullEvent, + Long debugEventsUntilDate) { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, FLAG_VALUE, + null, DEFAULT_VALUE, requireFullEvent, debugEventsUntilDate); + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DirectEventProcessorWithoutPersistenceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DirectEventProcessorWithoutPersistenceTest.java new file mode 100644 index 000000000..744440aee --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/DirectEventProcessorWithoutPersistenceTest.java @@ -0,0 +1,11 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.sdk.android.integrations.EventPersistence; + +/** {@link DirectEventProcessorTest} with persistence off, which is what an application gets by default. */ +public class DirectEventProcessorWithoutPersistenceTest extends DirectEventProcessorTest { + @Override + protected EventPersistence persistence() { + return EventPersistence.DISABLED; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventPersistenceBenchmark.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventPersistenceBenchmark.java new file mode 100644 index 000000000..7ed6e7817 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventPersistenceBenchmark.java @@ -0,0 +1,623 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assume.assumeTrue; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.AttributeRef; +import com.launchdarkly.sdk.ContextKind; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.internal.events.Event; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.IntConsumer; + +/** + * The Android counterpart of the iOS {@code EventPersistenceBenchmark}: what it costs to record an + * analytics event, so the encoding research can be argued from measurements on this platform rather + * than from the ones taken on iOS. + *

+ * This exists because the two platforms are not obviously comparable. iOS encodes with {@code Codable}, + * Android encodes with {@code EventOutputFormatter} in java-sdk-internal, and a JIT with escape + * analysis behaves nothing like ahead-of-time compiled Swift. The iOS finding -- that the cost is in + * the serializer's per-value machinery rather than in walking and redacting the context -- is a claim + * about {@code Codable}, and nothing about it transfers here by assumption. + *

+ * Skipped unless {@code LD_EVENT_BENCH=1}, because nothing here asserts anything a regression would + * trip and the timing loops take far longer than a unit test should: + * 

+ *   LD_EVENT_BENCH=1 ./gradlew :launchdarkly-android-client-sdk:testDebugUnitTest \
+ *       --tests '*EventPersistenceBenchmark*'
+ *
+ *

+ * Read the numbers as ratios, not absolutes. Three reasons. This runs on a desktop JVM rather + * than on ART on a device, which is the same caveat the iOS figures carry about running on a Mac. It + * is a hand-rolled timing loop rather than JMH, so it defends against the two failure modes that + * matter -- an unwarmed JIT and dead-code elimination -- and against nothing else. And the JVM's + * escape analysis can delete allocations here that a real caller would pay for, which flatters any + * row whose result is discarded. + */ +public class EventPersistenceBenchmark { + private static final String FLAG_KEY = "benchmark-flag"; + private static final int FLAG_VERSION = 7; + private static final int VARIATION = 1; + + /** + * Fixed rather than {@code System.currentTimeMillis()}, so that repeated runs encode identical + * bytes and the clock call is not charged to the encoder. The event-construction row below covers + * what the real path pays around it. + */ + private static final long CREATION_DATE = 1_700_000_000_000L; + + /** + * One already serialized event, byte for byte what the iOS benchmark stages, so the store figures on the + * two platforms are measuring the same amount of work reaching the same kind of file. + */ + private static final String SERIALIZED_EVENT = + "{\"kind\":\"feature\",\"key\":\"benchmark-flag\",\"value\":true,\"default\":false," + + "\"variation\":1,\"version\":7,\"creationDate\":1740000000000}"; + + /** How many times each timed loop is repeated; the fastest round is reported. */ + private static final int ROUNDS = 5; + + /** + * The sizes the commit-point figures are swept over. The same three are used by the iOS benchmark, so + * the two platforms' tables line up row for row. + */ + private static final int[] BULK_SIZES = {100, 1_000, 10_000}; + + /** + * Consumes results so that the JIT cannot prove the work is unused and delete it. Never read for + * its value. + */ + private static long blackhole; + + /** + * Holds an allocation so that escape analysis cannot scalar-replace it. Assigning to a static field + * is what makes the object genuinely exist, which is the difference between measuring an allocation + * and measuring nothing at all. + */ + private static Object escaped; + + @Rule + public TemporaryFolder eventsDirectory = new TemporaryFolder(); + + private final LDLogger logger = LDLogger.withAdapter(Logs.none(), ""); + + @Before + public void requireBenchmarking() { + assumeTrue("Set LD_EVENT_BENCH=1 to measure event recording", + "1".equals(System.getenv("LD_EVENT_BENCH"))); + } + + /** + * M1 and M2 for Android: where the time goes when one feature event is serialized, across the + * context shapes and privacy settings that change how much work redaction has to do. + *

+ * The shape of the result is the interesting part. If the cost is dominated by walking and + * redacting the context, it should grow with the number of attributes and grow again when private + * attributes force the filtering path. If it is dominated by the serializer's fixed per-event + * machinery, it should be far flatter than the attribute counts suggest. + */ + @Test + public void encodingCostByContextShape() { + List building = new ArrayList<>(); + List serializing = new ArrayList<>(); + + for (ContextShape shape : ContextShape.values()) { + final LDContext context = makeContext(shape, "benchmark-key"); + + building.add(new Measurement(shape.label, measure(200_000, i -> { + escaped = featureEvent(context, true); + }))); + + for (PrivacyShape privacy : privacyShapes()) { + final OutboundEventBuffer buffer = makeBuffer(privacy); + + byte[] sample = buffer.serialize(featureEvent(context, true)); + assertNotNull("the corpus produced an event that will not serialize", sample); + + double nanos = measure(50_000, i -> { + byte[] out = buffer.serialize(featureEvent(context, true)); + blackhole += out == null ? 0 : out.length; + }); + serializing.add(new Measurement( + shape.label + ", " + privacy.name + ", " + sample.length + " bytes", nanos)); + } + } + + report("building one feature event, no serialization", building); + report("serializing one feature event", serializing); + } + + /** + * The figures a customer actually pays, mirroring what {@link DirectEventProcessor} does per call. + *

+ * The three rows are the three paths through {@code recordEvaluationEvent} and {@code recordCustomEvent}: + * an evaluation of an untracked flag is a counter increment; an evaluation of a tracked flag also + * serializes and stages a full event; and a {@code track} is a commit point, which additionally turns + * the counters into summary events and writes everything to the log. + *

+ * The sequence is reproduced here rather than driven through the processor itself, which would need + * an HTTP server, a scheduler and a {@code ClientContext} whose costs would swamp what is being + * measured. It is kept deliberately close to the original; if the processor changes, this should too. + */ + @Test + public void recordingCostPerCall() throws IOException { + final OutboundEventBuffer buffer = makeBuffer(noRedaction()); + final EventStore store = EventStore.create(eventsDirectory.getRoot(), "benchmark-key", + "benchmark", Integer.MAX_VALUE, true, logger); + final LDContext context = makeContext(ContextShape.STUB, "benchmark-key"); + + try { + List results = new ArrayList<>(); + + results.add(new Measurement("evaluation, summary only", measure(200_000, i -> { + buffer.summarize(featureEvent(context, false)); + }))); + + results.add(new Measurement("evaluation, full event", measure(50_000, i -> { + Event.FeatureRequest event = featureEvent(context, true); + buffer.summarize(event); + byte[] out = buffer.serialize(event); + if (out != null) { + store.stage(out); + blackhole += out.length; + } + }))); + + results.add(new Measurement("evaluation + track (commit point)", measure(20_000, i -> { + buffer.summarize(featureEvent(context, false)); + byte[] out = buffer.serialize( + new Event.Custom(CREATION_DATE, "benchmark-event", context, LDValue.ofNull(), null)); + if (out != null) { + store.stage(out); + blackhole += out.length; + } + for (byte[] summary : buffer.serializeSummariesAndReset()) { + store.stage(summary, true); + blackhole += summary.length; + } + store.commit(); + }))); + + report("recording, per call", results); + } finally { + store.close(); + } + } + + /** + * What a commit point costs when it goes badly, rather than on average. + *

+ * {@code track} and {@code identify} commit on the caller's thread by design -- that is the durable + * barrier -- so this is the one figure here an application can observe as a stall. The mean is the number + * that gets quoted; the tail is the number a user notices. + *

+ * The counterpart iOS test runs this against a SQLite store as well, where the tail is the point: a + * database in WAL mode folds its write-ahead log back into the file every thousand pages, on whichever + * thread happens to be committing, so one unlucky caller pays for all of them. An append has no such + * thing, and these rows are what that claim is measured against. + *

+ * Two rows per size. {@code track} commits where it stands, so its tail is the write. The other row + * encodes and stages without committing, and the store never writes on its own, so its tail is the + * encode and the copy; the gap between the two is what the write itself costs. + *

+ * p99.9 only separates from the worst at ten thousand samples. Below that the index falls on the + * last sample and the two columns are the same number by construction. They are left side by side so the + * table lines up across the sweep, and so the size at which they part is visible rather than asserted. + */ + @Test + public void commitPointOutliers() throws IOException { + final byte[] serialized = SERIALIZED_EVENT.getBytes(StandardCharsets.UTF_8); + final OutboundEventBuffer buffer = makeBuffer(noRedaction()); + final LDContext context = makeContext(ContextShape.STUB, "benchmark-key"); + + // A store of its own for each size and each row, so that a log inherited from an earlier measurement + // does not charge the next one for the file it had already grown. + for (int calls : BULK_SIZES) { + final EventStore committing = EventStore.create(eventsDirectory.newFolder(), "benchmark-key", + "benchmark", Integer.MAX_VALUE, true, logger); + final EventStore staging = EventStore.create(eventsDirectory.newFolder(), "benchmark-key", + "benchmark", Integer.MAX_VALUE, true, logger); + try { + List results = new ArrayList<>(); + + results.add(new DistributionMeasurement("track, commits on this thread", + measureDistribution(calls, i -> { + committing.stage(serialized); + committing.commit(); + }))); + + results.add(new DistributionMeasurement("evaluation, staged only", + measureDistribution(calls, i -> { + Event.FeatureRequest event = featureEvent(context, true); + buffer.summarize(event); + byte[] out = buffer.serialize(event); + if (out != null) { + staging.stage(out); + blackhole += out.length; + } + }))); + + reportDistribution("one call, " + count(calls) + " of them, on the caller's thread", results); + } finally { + committing.close(); + staging.close(); + } + } + } + + /** + * Whether O2 -- caching the encoded context and reusing it while the context has not changed -- + * could work here, which comes down to what the cache key would cost to check. + *

+ * Android differs from iOS in a way that matters. {@code LDContext} is an immutable object passed by + * reference, and nothing on the event path copies it, so an application that holds one context hands + * the same instance to every event and {@code equals} can settle on reference identity. iOS had to + * reach that fast path by not rebuilding a value type. The row that decides the question is the other + * one: an application that rebuilds its context between evaluations produces an equal object sharing + * nothing, and that comparison is the price of a cache hit in the worst case. + */ + @Test + public void contextComparisonCost() { + List comparisons = new ArrayList<>(); + List hashing = new ArrayList<>(); + + for (ContextShape shape : ContextShape.values()) { + final LDContext context = makeContext(shape, "benchmark-key"); + final LDContext sameInstance = context; + final LDContext rebuilt = makeContext(shape, "benchmark-key"); + + // Both must be equal, or the figures below measure an early exit rather than a comparison. + assertEquals(shape.label, context, sameInstance); + assertEquals(shape.label, context, rebuilt); + + comparisons.add(new Measurement("same instance, " + shape.label, measure(500_000, i -> { + blackhole += context.equals(sameInstance) ? 1 : 0; + }))); + comparisons.add(new Measurement("independently built, " + shape.label, measure(200_000, i -> { + blackhole += context.equals(rebuilt) ? 1 : 0; + }))); + hashing.add(new Measurement(shape.label, measure(500_000, i -> { + blackhole += context.hashCode(); + }))); + } + + report("comparing two equal contexts", comparisons); + report("hashing a context", hashing); + } + + /** + * What the deferred half of the bill costs: summaries are not serialized per evaluation but at + * commit points and deliveries, and the size of that payment depends on how many distinct flags the + * counters have accumulated. + *

+ * Counting and serializing are reported separately, because only the second is deferred -- the + * counter increment is already paid on every evaluation. + */ + @Test + public void summarySerializationCost() { + final LDContext context = makeContext(ContextShape.STUB, "benchmark-key"); + List results = new ArrayList<>(); + + for (final int flagCount : new int[] {1, 10, 50}) { + final OutboundEventBuffer buffer = makeBuffer(noRedaction()); + + double countingOnly = measure(20_000, i -> { + for (int flag = 0; flag < flagCount; flag++) { + buffer.summarize(summaryEvent(context, flag)); + } + }); + // Drained, so the counters left behind above do not inflate the first serialized round. + buffer.serializeSummariesAndReset(); + + double countingAndSerializing = measure(20_000, i -> { + for (int flag = 0; flag < flagCount; flag++) { + buffer.summarize(summaryEvent(context, flag)); + } + for (byte[] summary : buffer.serializeSummariesAndReset()) { + blackhole += summary.length; + } + }); + + results.add(new Measurement(flagCount + " flags: counting only", countingOnly)); + results.add(new Measurement(flagCount + " flags: counting + serializing", + countingAndSerializing)); + results.add(new Measurement(flagCount + " flags: serializing alone", + countingAndSerializing - countingOnly)); + } + + report("turning counters into a summary event", results); + } + + // MARK: Corpus + + /** + * The same five shapes the iOS benchmark uses, so the two sets of numbers describe the same + * contexts. + */ + private enum ContextShape { + KEY_ONLY("key only"), + STUB("stub, 9 attributes"), + WIDE("20 flat attributes"), + NESTED("nested attributes"), + MULTI("multi-context"); + + final String label; + + ContextShape(String label) { + this.label = label; + } + } + + private static LDContext makeContext(ContextShape shape, String key) { + switch (shape) { + case KEY_ONLY: + return LDContext.create(key); + case STUB: + return stubContext(key); + case WIDE: { + com.launchdarkly.sdk.ContextBuilder builder = LDContext.builder(key).name("Wide"); + for (int i = 0; i < 20; i++) { + builder.set("attribute" + i, "value" + i); + } + return builder.build(); + } + case NESTED: + return LDContext.builder(key) + .name("Nested") + .set("address", LDValue.buildObject() + .put("street", "1 Main St") + .put("city", "Springfield") + .put("geo", LDValue.buildObject().put("lat", 1.5).put("lon", -2.5).build()) + .build()) + .set("tags", LDValue.buildArray() + .add("a").add("b").add("c").add("d").add("e").build()) + .build(); + case MULTI: + return LDContext.createMulti( + stubContext(key), + LDContext.builder(ContextKind.of("device"), "device-" + key) + .set("os", LDValue.buildObject() + .put("name", "Android").put("version", 14).build()) + .build()); + default: + throw new IllegalArgumentException(shape.name()); + } + } + + /** Nine attributes, matching the shape iOS calls its stub context. */ + private static LDContext stubContext(String key) { + return LDContext.builder(key) + .name("Stub Context") + .set("firstName", "Ada") + .set("lastName", "Lovelace") + .set("email", "ada@example.com") + .set("country", "US") + .set("age", 36) + .set("score", 12.5) + .set("verified", true) + .set("plan", "enterprise") + .build(); + } + + /** The privacy settings that change what redaction has to do. */ + private static final class PrivacyShape { + final String name; + final boolean allAttributesPrivate; + final Collection privateAttributes; + + PrivacyShape(String name, boolean allAttributesPrivate, Collection privateAttributes) { + this.name = name; + this.allAttributesPrivate = allAttributesPrivate; + this.privateAttributes = privateAttributes; + } + } + + private static PrivacyShape noRedaction() { + return new PrivacyShape("no redaction", false, Collections.emptyList()); + } + + private static List privacyShapes() { + return Arrays.asList( + noRedaction(), + new PrivacyShape("1 global private", false, + Collections.singletonList(AttributeRef.fromLiteral("email"))), + new PrivacyShape("all private", true, Collections.emptyList())); + } + + private OutboundEventBuffer makeBuffer(PrivacyShape privacy) { + // Unbounded cardinality, so a benchmark that sweeps context counts measures the encoder + // rather than the point at which the summarizer starts turning contexts away. + return new OutboundEventBuffer(privacy.allAttributesPrivate, privacy.privateAttributes, + true, Integer.MAX_VALUE, logger); + } + + private static Event.FeatureRequest featureEvent(LDContext context, boolean requireFullEvent) { + return new Event.FeatureRequest(CREATION_DATE, FLAG_KEY, context, FLAG_VERSION, VARIATION, + LDValue.of(true), LDValue.of(false), null, null, requireFullEvent, null, false); + } + + private static Event.FeatureRequest summaryEvent(LDContext context, int flagIndex) { + return new Event.FeatureRequest(CREATION_DATE, FLAG_KEY + "-" + flagIndex, context, + FLAG_VERSION, VARIATION, LDValue.of(true), LDValue.of(false), null, null, false, null, + false); + } + + // MARK: Harness + + /** + * Times {@code body}, returning nanoseconds per iteration. + *

+ * Warmed first, because a JVM measured cold reports the interpreter rather than the compiled code. + * Then run several times with the fastest round reported: a garbage collection or a descheduled + * thread can only ever make a round slower, so the minimum is the closest estimate of the work + * itself. That is also why these numbers should not be read as what a busy device would see. + */ + private double measure(int iterations, IntConsumer body) { + int warmup = Math.max(10_000, iterations / 4); + for (int i = 0; i < warmup; i++) { + body.accept(i); + } + + double best = Double.MAX_VALUE; + for (int round = 0; round < ROUNDS; round++) { + long start = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + body.accept(i); + } + long elapsed = System.nanoTime() - start; + best = Math.min(best, (double) elapsed / iterations); + } + return best; + } + + /** How long one call took, across many of them, when the tail is the point. */ + private static final class Distribution { + final double median; + final double mean; + final double p999; + final double worst; + + Distribution(double median, double mean, double p999, double worst) { + this.median = median; + this.mean = mean; + this.p999 = p999; + this.worst = worst; + } + } + + /** + * Times every call rather than the enclosing loop, so the tail survives to be reported. + *

+ * One round, and the median rather than the fastest. Every other figure here reports the cheapest of + * several rounds, on the reasoning that noise can only make a round slower -- but here an outlier is the + * measurement, and that aggregation would throw away the answer. + */ + private Distribution measureDistribution(int iterations, IntConsumer body) { + int warmup = Math.max(10_000, iterations / 4); + for (int i = 0; i < warmup; i++) { + body.accept(i); + } + + long[] samples = new long[iterations]; + for (int i = 0; i < iterations; i++) { + long start = System.nanoTime(); + body.accept(i); + samples[i] = System.nanoTime() - start; + } + + long total = 0; + for (long sample : samples) { + total += sample; + } + Arrays.sort(samples); + return new Distribution(samples[iterations / 2], + (double) total / iterations, + samples[Math.min(iterations - 1, (int) ((long) iterations * 999 / 1000))], + samples[iterations - 1]); + } + + private static final class DistributionMeasurement { + final String name; + final Distribution distribution; + + DistributionMeasurement(String name, Distribution distribution) { + this.name = name; + this.distribution = distribution; + } + } + + private static final class Measurement { + final String name; + final double nanosPerOp; + + Measurement(String name, double nanosPerOp) { + this.name = name; + this.nanosPerOp = nanosPerOp; + } + } + + private static void report(String title, List results) { + int width = 0; + for (Measurement result : results) { + width = Math.max(width, result.name.length()); + } + StringBuilder out = new StringBuilder("\n").append(title).append('\n'); + for (Measurement result : results) { + out.append(" ").append(pad(result.name, width)).append(" ") + .append(format(result.nanosPerOp)).append('\n'); + } + System.out.print(out); + System.out.flush(); + } + + private static void reportDistribution(String title, List results) { + int width = 0; + for (DistributionMeasurement result : results) { + width = Math.max(width, result.name.length()); + } + StringBuilder out = new StringBuilder("\n").append(title).append('\n') + .append(" ").append(pad("", width)) + .append(" ").append(column("median")) + .append(" ").append(column("mean")) + .append(" ").append(column("p99.9")) + .append(" ").append(column("worst")).append('\n'); + for (DistributionMeasurement result : results) { + out.append(" ").append(pad(result.name, width)) + .append(" ").append(format(result.distribution.median)) + .append(" ").append(format(result.distribution.mean)) + .append(" ").append(format(result.distribution.p999)) + .append(" ").append(format(result.distribution.worst)).append('\n'); + } + System.out.print(out); + System.out.flush(); + } + + /** Renders a count with thousands separators, so a table title reads as a quantity, not a code literal. */ + private static String count(int value) { + return String.format("%,d", value); + } + + /** Right-aligns a column heading over the fixed width {@link #format} produces. */ + private static String column(String title) { + StringBuilder padded = new StringBuilder(); + for (int i = title.length(); i < 11; i++) { + padded.append(' '); + } + return padded.append(title).toString(); + } + + private static String pad(String value, int width) { + StringBuilder padded = new StringBuilder(value); + while (padded.length() < width) { + padded.append(' '); + } + return padded.toString(); + } + + private static String format(double nanoseconds) { + if (nanoseconds >= 1_000_000) { + return String.format("%8.2f ms", nanoseconds / 1_000_000); + } + if (nanoseconds >= 1_000) { + return String.format("%8.2f \u00b5s", nanoseconds / 1_000); + } + return String.format("%8.1f ns", nanoseconds); + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorBufferingTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorBufferingTest.java new file mode 100644 index 000000000..8d9ee9e0d --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorBufferingTest.java @@ -0,0 +1,339 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; +import com.launchdarkly.testhelpers.httptest.HttpServer; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * Tests for what the event processor is allowed to drop when flag evaluations arrive faster than + * it can consume them. + *

+ * The processor used to be a thin wrapper around java-sdk-internal's + * {@code DefaultEventProcessor}, which buffers in two stages: every recorded event is first + * offered to a bounded "inbox" queue, and only after a background dispatcher thread picks it up is + * it either summarized or placed in the outbox that a flush actually drains. The inbox is sized to + * the configured event capacity, and it carries the processor's own FLUSH and SHUTDOWN control + * messages as well as events. So a burst of evaluations could fill it and cause silent drops even + * though those evaluations were only ever going to contribute a counter to a summary event, and + * even though unrelated custom events were competing for the same slots. + *

+ * These tests pin the behavior we want instead: summarization happens when the evaluation is + * recorded, so an evaluation never occupies a queue slot, and capacity only ever limits + * full-fidelity events. With summarization ahead of buffering there is no queue in this path, so + * the counts asserted below are exact rather than approximate. + */ +public class EventProcessorBufferingTest extends EventProcessorTestBase { + private static final String FLAG_KEY = "burst-flag"; + private static final int FLAG_VERSION = 10; + private static final int VARIATION = 1; + private static final LDValue FLAG_VALUE = LDValue.of(true); + private static final LDValue DEFAULT_VALUE = LDValue.of(false); + + // Deliberately far smaller than the number of evaluations each test records, so that any + // capacity-limited queue in the evaluation path would be guaranteed to overflow. + private static final int CAPACITY = 100; + + // For the tests that fill the summarizer to its context limit. Each counted context becomes its + // own summary event, so the flush is roughly this many summaries; at CAPACITY that came to about + // 28 KB, past what the test server reliably records with its single unlooped read of the body. + private static final int CONTEXT_LIMIT = 10; + + private static final int BURST_THREADS = 4; + private static final int EVALUATIONS_PER_THREAD = 25_000; + private static final int TOTAL_EVALUATIONS = BURST_THREADS * EVALUATIONS_PER_THREAD; + + // How many full-fidelity events the interleaving tests space through the burst. Comfortably + // under CAPACITY, so the buffer is never legitimately entitled to drop any of them. + private static final int INTERLEAVED_EVENTS = 50; + + @Test + public void summaryCountsSurviveEvaluationBurst() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CAPACITY); + try { + new EvaluationBurst(eventProcessor).join(); + + List events = flushAndCollect(eventProcessor, server); + + // Every evaluation contributes to the summary regardless of capacity, because a + // summary counter is not a buffered event. + assertEquals(TOTAL_EVALUATIONS, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void customEventsSurviveEvaluationBurst() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CAPACITY); + try { + // Record custom events spaced evenly through the burst. These are full-fidelity + // events and there are far fewer of them than the configured capacity, so none of + // them may be lost no matter how many evaluations are happening alongside them. + EvaluationBurst burst = new EvaluationBurst(eventProcessor); + for (int i = 1; i <= INTERLEAVED_EVENTS; i++) { + burst.awaitFraction(i, INTERLEAVED_EVENTS + 1); + eventProcessor.recordCustomEvent(CONTEXT, "burst-custom", LDValue.ofNull(), null); + } + burst.join(); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(INTERLEAVED_EVENTS, countEventsOfKind(events, "custom")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void fullFidelityEvaluationEventsSurviveEvaluationBurst() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CAPACITY); + try { + // A flag with trackEvents on produces a full feature event in addition to its + // summary counter. Evaluations of untracked flags must not crowd these out. + EvaluationBurst burst = new EvaluationBurst(eventProcessor); + for (int i = 1; i <= INTERLEAVED_EVENTS; i++) { + burst.awaitFraction(i, INTERLEAVED_EVENTS + 1); + eventProcessor.recordEvaluationEvent(CONTEXT, "tracked-flag", FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, true, null); + } + burst.join(); + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(INTERLEAVED_EVENTS, countEventsOfKind(events, "feature")); + // The tracked evaluations are summarized too, alongside the burst. + assertEquals(INTERLEAVED_EVENTS, summaryCountFor(events, "tracked-flag")); + assertEquals(TOTAL_EVALUATIONS, summaryCountFor(events, FLAG_KEY)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void distinctSummarizedContextsAreBoundedByCapacity() throws Exception { + // A summary counter costs a retained context and a set of counters per distinct context, and + // nothing drains it while the client is offline. Capacity has to bound that, or an application + // that goes on identifying through an outage grows the summarizer for as long as it lasts. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CONTEXT_LIMIT); + try { + for (int i = 0; i < CONTEXT_LIMIT * 3; i++) { + eventProcessor.recordEvaluationEvent(contextNumber(i), FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + + List events = flushAndCollect(eventProcessor, server); + + // One summary event per context that was counted, so this is the cardinality the + // summarizer was holding. + assertEquals(CONTEXT_LIMIT, countEventsOfKind(events, "summary")); + logging.assertWarnLogged("Exceeded the number of contexts"); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void exceedingContextLimitIsWarnedOncePerFlushRun() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CONTEXT_LIMIT); + try { + for (int i = 0; i < CONTEXT_LIMIT * 3; i++) { + eventProcessor.recordEvaluationEvent(contextNumber(i), FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + flushAndCollect(eventProcessor, server); + + for (int i = 0; i < CONTEXT_LIMIT * 3; i++) { + eventProcessor.recordEvaluationEvent(contextNumber(100 + i), FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + flushAndCollect(eventProcessor, server); + + int warnCount = 0; + for (String msg : logging.logCapture.getMessageStrings()) { + if (msg.contains("Exceeded the number of contexts")) { + warnCount++; + } + } + assertEquals(2, warnCount); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aContextAlreadyBeingCountedKeepsCountingOnceTheLimitIsReached() throws Exception { + // The limit is on how many contexts are held, not on how many evaluations are counted. An + // application evaluating against one context must not start losing counts because some other + // part of it churned through contexts. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CONTEXT_LIMIT); + try { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, null, DEFAULT_VALUE, false, null); + for (int i = 0; i < CONTEXT_LIMIT * 3; i++) { + eventProcessor.recordEvaluationEvent(contextNumber(i), FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + int evaluationsAfterTheLimit = 500; + for (int i = 0; i < evaluationsAfterTheLimit; i++) { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(evaluationsAfterTheLimit + 1, summaryCountForContext(events, CONTEXT)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void theLimitLiftsOnceTheSummariesHaveBeenDelivered() throws Exception { + // The bound is on how many contexts are held at once, not on how many an application may ever + // use, so a delivery has to make room for the next set. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CONTEXT_LIMIT); + try { + for (int i = 0; i < CONTEXT_LIMIT * 3; i++) { + eventProcessor.recordEvaluationEvent(contextNumber(i), FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + flushAndCollect(eventProcessor, server); + + for (int i = CONTEXT_LIMIT * 3; i < CONTEXT_LIMIT * 4; i++) { + eventProcessor.recordEvaluationEvent(contextNumber(i), FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + } + List events = flushAndCollect(eventProcessor, server); + + assertEquals(CONTEXT_LIMIT, countEventsOfKind(events, "summary")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aCommitPointMakesRoomInTheSummarizer() throws Exception { + assumePersisting(); + // Counters are materialized into summary events and reset at every commit point, so the + // summarizer only ever holds the contexts seen since the last one. That reset is what keeps + // the cardinality limit from binding: without it an application identifying its way through + // more contexts than capacity would start losing counts partway through. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, CAPACITY); + try { + int contexts = CAPACITY * 3; + for (int i = 0; i < contexts; i++) { + LDContext context = contextNumber(i); + eventProcessor.recordEvaluationEvent(context, FLAG_KEY, FLAG_VERSION, VARIATION, + FLAG_VALUE, null, DEFAULT_VALUE, false, null); + eventProcessor.recordIdentifyEvent(context); // a commit point + } + + List events = flushAndCollect(eventProcessor, server); + + assertEquals(contexts, countEventsOfKind(events, "summary")); + assertFalse("evaluations were turned away even though every one of them followed a" + + " commit point that should have made room", + logged("Exceeded the number of contexts")); + } finally { + eventProcessor.close(); + } + } + } + + private boolean logged(String messageSubstring) { + for (String message : logging.logCapture.getMessageStrings()) { + if (message.contains(messageSubstring)) { + return true; + } + } + return false; + } + + private static LDContext contextNumber(int i) { + return LDContext.builder("burst-user-" + i).set("email", "user" + i + "@example.com").build(); + } + + /** @return how many evaluations the summary for this context counted, across every flag in it */ + private static int summaryCountForContext(List events, LDContext context) { + int total = 0; + for (LDValue event : events) { + if (!"summary".equals(event.get("kind").stringValue()) + || !context.getKey().equals(event.get("context").get("key").stringValue())) { + continue; + } + LDValue features = event.get("features"); + for (String flagKey : features.keys()) { + for (LDValue counter : features.get(flagKey).get("counters").values()) { + total += counter.get("count").intValue(); + } + } + } + return total; + } + + /** + * Records evaluations of an untracked flag from several threads at once, faster than any + * single consumer thread could drain them. Tests that need to interleave other events use + * {@link #awaitFraction} rather than sleeping, so the interleaved events land at the same + * points in the burst regardless of how fast the machine running the test is. + */ + private final class EvaluationBurst { + private final List workers = new ArrayList<>(); + private volatile int progress; + + EvaluationBurst(EventProcessor eventProcessor) { + for (int i = 0; i < BURST_THREADS; i++) { + boolean reportsProgress = i == 0; + Thread worker = new Thread(() -> { + for (int j = 0; j < EVALUATIONS_PER_THREAD; j++) { + eventProcessor.recordEvaluationEvent(CONTEXT, FLAG_KEY, FLAG_VERSION, + VARIATION, FLAG_VALUE, null, DEFAULT_VALUE, false, null); + if (reportsProgress) { + progress = j + 1; + } + } + }); + worker.start(); + workers.add(worker); + } + } + + /** Blocks until the burst is {@code numerator/denominator} of the way through. */ + void awaitFraction(int numerator, int denominator) { + int target = (int) ((long) EVALUATIONS_PER_THREAD * numerator / denominator); + while (progress < target && workers.get(0).isAlive()) { + Thread.yield(); + } + } + + void join() throws InterruptedException { + for (Thread worker : workers) { + worker.join(); + } + } + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorBufferingWithoutPersistenceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorBufferingWithoutPersistenceTest.java new file mode 100644 index 000000000..4429958cd --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorBufferingWithoutPersistenceTest.java @@ -0,0 +1,11 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.sdk.android.integrations.EventPersistence; + +/** {@link EventProcessorBufferingTest} with persistence off, which is what an application gets by default. */ +public class EventProcessorBufferingWithoutPersistenceTest extends EventProcessorBufferingTest { + @Override + protected EventPersistence persistence() { + return EventPersistence.DISABLED; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlagsTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlagsTest.java new file mode 100644 index 000000000..31f66e523 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlagsTest.java @@ -0,0 +1,176 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.internal.events.Event; +import com.launchdarkly.testhelpers.httptest.HttpServer; + +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Tests for the two per-event flags the SDK is expected to honour: {@code excludeFromSummaries}, + * which keeps an evaluation out of the counters, and {@code samplingRatio}, which decides whether a + * full event is kept at all. + *

+ * The two are enforced at different stages, because that is where each one's subject matter lives. + * The counters are in {@link OutboundEventBuffer}, so the exclusion is checked there. The run of + * full events and the capacity counted against it are in {@link DirectEventProcessor}, so sampling + * is checked there, ahead of the capacity check that would otherwise record a sampled-out event as + * a drop. + *

+ * Neither flag can be reached through the SDK's own recording path. {@link DirectEventProcessor} + * builds its feature events with the constructor overload that fixes them at {@code false} and + * {@code 1}, so every evaluation an application performs leaves both at their defaults. That is why + * these tests hand an event straight to the stage that enforces the flag rather than going through + * {@code recordEvaluationEvent}: those are the only seams at which the flags can vary. + *

+ * They are worth pinning even so. java-sdk-internal's {@code DefaultEventProcessor}, which this + * path replaced, honoured both, and anything that later constructs feature events through the + * fourteen-argument constructor — adopting sampling is the obvious candidate — makes both live at + * once. The failure would be silent and would show up as wrong analytics rather than as an error, + * since an event carries its sampling ratio on the wire so the receiver can scale by it. + *

+ * Both ratios used here are deterministic rather than probabilistic: {@code Sampler.shouldSample} + * short-circuits to {@code true} at 1 and to {@code false} at 0, so nothing below is flaky. + */ +public class EventProcessorFlagsTest extends EventProcessorTestBase { + private static final String FLAG_KEY = "flags-test-flag"; + private static final int FLAG_VERSION = 3; + private static final int VARIATION = 1; + private static final LDValue FLAG_VALUE = LDValue.of(true); + private static final LDValue DEFAULT_VALUE = LDValue.of(false); + private static final long CREATION_DATE = 1_700_000_000_000L; + + // Far more than any test here records, so that capacity can never be what discarded an event. + private static final int CAPACITY = 100; + + @Test + public void anEvaluationExcludedFromSummariesIsNotCounted() { + OutboundEventBuffer buffer = makeBuffer(); + + buffer.summarize(evaluation(true)); + + assertTrue("an excluded evaluation should leave nothing to send", + drainToEvents(buffer).isEmpty()); + } + + @Test + public void anEvaluationNotExcludedFromSummariesIsCounted() { + OutboundEventBuffer buffer = makeBuffer(); + + buffer.summarize(evaluation(false)); + + assertEquals(1, summaryCountFor(drainToEvents(buffer), FLAG_KEY)); + } + + @Test + public void excludingOneEvaluationLeavesTheOthersCounted() { + OutboundEventBuffer buffer = makeBuffer(); + + buffer.summarize(evaluation(false)); + buffer.summarize(evaluation(true)); + buffer.summarize(evaluation(false)); + + // The exclusion applies to the event that carried it and to nothing else, which is what + // distinguishes honouring the flag from dropping the whole summary. + assertEquals(2, summaryCountFor(drainToEvents(buffer), FLAG_KEY)); + } + + @Test + public void aFullEventSampledOutIsNotKept() throws Exception { + try (HttpServer server = startEventsServer()) { + DirectEventProcessor processor = processor(server); + try { + processor.record(evaluation(false, 0)); + // Something the SDK will certainly send, so the flush has a payload to look at. + // Without it there would be no request at all, and nothing to distinguish an event + // that was dropped from one that simply has not been sent yet. + processor.recordIdentifyEvent(CONTEXT); + + List events = flushAndCollect(processor, server); + + assertEquals(0, countEventsOfKind(events, "feature")); + assertEquals(1, countEventsOfKind(events, "identify")); + } finally { + processor.close(); + } + } + } + + @Test + public void aFullEventSampledInIsKept() throws Exception { + try (HttpServer server = startEventsServer()) { + DirectEventProcessor processor = processor(server); + try { + processor.record(evaluation(false, 1)); + + assertEquals(1, countEventsOfKind(flushAndCollect(processor, server), "feature")); + } finally { + processor.close(); + } + } + } + + @Test + public void stagingAnEventDoesNotDrawForSamplingASecondTime() throws Exception { + // The processor draws once, as the event is recorded, and hands on only what it kept. A second + // draw where the event is serialized would square the odds against anything with a ratio above + // one, and would drop it without counting it anywhere. A ratio of 0 stands in for a draw that + // comes out against the event, because Sampler.shouldSample refuses it every time. + OutboundEventBuffer buffer = makeBuffer(); + Event sampledOut = evaluation(false, 0); + + assertEquals(1, buffer.serializeAll(Collections.singletonList((Event) sampledOut)).size()); + assertNotNull(buffer.serialize(sampledOut)); + } + + @Test + public void aSampledOutEventIsNotCountedAsADrop() throws Exception { + try (HttpServer server = startEventsServer()) { + DirectEventProcessor processor = processor(server); + try { + processor.record(evaluation(false, 0)); + + // Sampling and capacity are different reasons not to keep an event, and only the + // second is a loss the SDK owes anyone a count of. + assertEquals(0, processor.getAndClearDroppedCount()); + } finally { + processor.close(); + } + } + } + + private DirectEventProcessor processor(HttpServer server) { + return (DirectEventProcessor) makeEventProcessor(server, CAPACITY); + } + + private OutboundEventBuffer makeBuffer() { + return new OutboundEventBuffer(false, Collections.emptyList(), false, 100, logging.logger); + } + + private Event.FeatureRequest evaluation(boolean excludeFromSummaries) { + return evaluation(excludeFromSummaries, 1); + } + + private Event.FeatureRequest evaluation(boolean excludeFromSummaries, long samplingRatio) { + return new Event.FeatureRequest(CREATION_DATE, FLAG_KEY, CONTEXT, FLAG_VERSION, VARIATION, + FLAG_VALUE, DEFAULT_VALUE, null, null, true, null, false, samplingRatio, + excludeFromSummaries); + } + + private List drainToEvents(OutboundEventBuffer buffer) { + List events = new ArrayList<>(); + for (byte[] summary : buffer.serializeSummariesAndReset()) { + events.add(LDValue.parse(new String(summary, StandardCharsets.UTF_8))); + } + return events; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlagsWithoutPersistenceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlagsWithoutPersistenceTest.java new file mode 100644 index 000000000..a63a323f7 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlagsWithoutPersistenceTest.java @@ -0,0 +1,11 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.sdk.android.integrations.EventPersistence; + +/** {@link EventProcessorFlagsTest} with persistence off, which is what an application gets by default. */ +public class EventProcessorFlagsWithoutPersistenceTest extends EventProcessorFlagsTest { + @Override + protected EventPersistence persistence() { + return EventPersistence.DISABLED; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlushAsyncDefaultTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlushAsyncDefaultTest.java new file mode 100644 index 000000000..427fc39a9 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorFlushAsyncDefaultTest.java @@ -0,0 +1,90 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.launchdarkly.sdk.EvaluationReason; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Covers what {@link EventProcessor#flushAsync()} does for an implementation that predates it and + * has only its unbounded {@link EventProcessor#blockingFlush()}. The SDK has to be able to put a + * deadline on such a flush, and must not tell the caller its events are safe without knowing. + */ +public class EventProcessorFlushAsyncDefaultTest { + @Rule + public Timeout globalTimeout = Timeout.seconds(30); + + @Test + public void theCallersDeadlineBoundsTheWaitAndNotTheFlush() throws Exception { + LegacyEventProcessor eventProcessor = new LegacyEventProcessor(); + Future delivery = eventProcessor.flushAsync(); + + try { + delivery.get(100, TimeUnit.MILLISECONDS); + fail("the wait outlived the deadline"); + } catch (TimeoutException expected) { + // The flush is still going, which is why this is what the caller is told. + } + assertFalse(eventProcessor.flushReturned.get()); + + eventProcessor.letFlushFinish.countDown(); + assertTrue("the flush's own outcome was not reported", + delivery.get(5, TimeUnit.SECONDS)); + assertTrue(eventProcessor.flushReturned.get()); + } + + /** An implementation written before {@code flushAsync} existed. */ + private static final class LegacyEventProcessor implements EventProcessor { + final CountDownLatch letFlushFinish = new CountDownLatch(1); + final AtomicBoolean flushReturned = new AtomicBoolean(false); + + @Override + public void blockingFlush() { + try { + letFlushFinish.await(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + flushReturned.set(true); + } + + @Override + public void flush() {} + + @Override + public void setInBackground(boolean inBackground) {} + + @Override + public void setOffline(boolean offline) {} + + @Override + public void close() {} + + @Override + public void recordEvaluationEvent(LDContext context, String flagKey, int flagVersion, + int variation, LDValue value, EvaluationReason reason, + LDValue defaultValue, boolean requireFullEvent, + Long debugEventsUntilDate) {} + + @Override + public void recordIdentifyEvent(LDContext context) {} + + @Override + public void recordCustomEvent(LDContext context, String eventKey, LDValue data, + Double metricValue) {} + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorPrivacyTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorPrivacyTest.java new file mode 100644 index 000000000..5ff0bcba3 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorPrivacyTest.java @@ -0,0 +1,269 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.sdk.ContextKind; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; +import com.launchdarkly.testhelpers.httptest.HttpServer; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * Context attribute redaction, as observed in the payload the processor actually posts. + *

+ * Redaction is not reimplemented by the SDK's event processor - it reuses java-sdk-internal's + * context formatter through {@link OutboundEventBuffer}. These + * tests exist because that reuse is the whole argument for the current design, and nothing else in + * this repository asserts that an attribute an application marked private stays out of the wire + * format. Beyond the redaction itself they pin the two rules that are easy to get wrong when + * events are assembled somewhere new: which events inline a context at all, and which of them + * additionally redact the attributes of an anonymous context. + */ +public class EventProcessorPrivacyTest extends EventProcessorTestBase { + private static final String FLAG_KEY = "flag-key"; + private static final int DEFAULT_CAPACITY = 100; + + private static final LDContext PERSON = LDContext.builder("user-key") + .name("Sandy") + .set("email", "sandy@example.com") + .set("address", LDValue.buildObject() + .put("city", "Oakland") + .put("street", "123 Main St") + .build()) + .build(); + + @Test + public void namedPrivateAttributesAreRedactedAndReported() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).privateAttributes("email"), true); + try { + eventProcessor.recordIdentifyEvent(PERSON); + + LDValue context = requireEventOfKind(flushAndCollect(eventProcessor, server), + "identify").get("context"); + + assertEquals(LDValue.of("user-key"), context.get("key")); + assertEquals("email must not appear in the payload", + LDValue.ofNull(), context.get("email")); + // Attributes that were not marked private are still sent. + assertEquals(LDValue.of("Sandy"), context.get("name")); + assertEquals(redacted("email"), redactedAttributes(context)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void allAttributesPrivateLeavesOnlyTheKey() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).allAttributesPrivate(true), true); + try { + eventProcessor.recordIdentifyEvent(PERSON); + + LDValue context = requireEventOfKind(flushAndCollect(eventProcessor, server), + "identify").get("context"); + + // The key and kind are identifiers rather than attributes, so they survive. + assertEquals(LDValue.of("user-key"), context.get("key")); + assertEquals(LDValue.of("user"), context.get("kind")); + assertEquals(LDValue.ofNull(), context.get("name")); + assertEquals(LDValue.ofNull(), context.get("email")); + assertEquals(LDValue.ofNull(), context.get("address")); + assertEquals(redacted("address", "email", "name"), redactedAttributes(context)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void aPrivateSubAttributeRedactsOnlyThatProperty() throws Exception { + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).privateAttributes("/address/street"), true); + try { + eventProcessor.recordIdentifyEvent(PERSON); + + LDValue context = requireEventOfKind(flushAndCollect(eventProcessor, server), + "identify").get("context"); + + assertEquals(LDValue.of("Oakland"), context.get("address").get("city")); + assertEquals(LDValue.ofNull(), context.get("address").get("street")); + assertEquals(redacted("/address/street"), redactedAttributes(context)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void attributesMarkedPrivateOnTheContextItselfAreRedacted() throws Exception { + // An application can mark an attribute private per-context instead of globally; the + // processor is not configured with anything in this case. + LDContext context = LDContext.builder("user-key") + .name("Sandy") + .set("email", "sandy@example.com") + .privateAttributes("email") + .build(); + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordIdentifyEvent(context); + + LDValue delivered = requireEventOfKind(flushAndCollect(eventProcessor, server), + "identify").get("context"); + + assertEquals(LDValue.ofNull(), delivered.get("email")); + assertEquals(LDValue.of("Sandy"), delivered.get("name")); + assertEquals(redacted("email"), redactedAttributes(delivered)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void anonymousContextAttributesAreRedactedFromFeatureEventsButNotIdentifyEvents() throws Exception { + LDContext anonymous = LDContext.builder(ContextKind.DEFAULT, "anon-key") + .anonymous(true) + .set("email", "sandy@example.com") + .build(); + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, DEFAULT_CAPACITY); + try { + eventProcessor.recordIdentifyEvent(anonymous); + eventProcessor.recordEvaluationEvent(anonymous, FLAG_KEY, 10, 1, LDValue.of(true), + null, LDValue.of(false), true, null); + + List events = flushAndCollect(eventProcessor, server); + + // An identify event is the application deliberately registering this context. + LDValue identifyContext = requireEventOfKind(events, "identify").get("context"); + assertEquals(LDValue.of("sandy@example.com"), identifyContext.get("email")); + + // A feature event redacts every attribute of an anonymous context, without the + // application having to mark anything private. + LDValue featureContext = requireEventOfKind(events, "feature").get("context"); + assertEquals(LDValue.of("anon-key"), featureContext.get("key")); + assertEquals(LDValue.ofNull(), featureContext.get("email")); + assertEquals(redacted("email"), redactedAttributes(featureContext)); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void summaryEventsRedactPrivateAttributesOfTheContextTheyInline() throws Exception { + // Because this SDK summarizes per context, a summary event inlines the context it counted + // rather than just its key, and so it is subject to redaction like any other event. This + // is the case most easily missed, since summaries are produced by the buffer rather than + // by an explicit record call. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).privateAttributes("email"), true); + try { + eventProcessor.recordEvaluationEvent(PERSON, FLAG_KEY, 10, 1, LDValue.of(true), + null, LDValue.of(false), false, null); + + LDValue summary = requireEventOfKind(flushAndCollect(eventProcessor, server), + "summary"); + + LDValue context = summary.get("context"); + assertEquals(LDValue.of("user-key"), context.get("key")); + assertEquals(LDValue.ofNull(), context.get("email")); + assertEquals(redacted("email"), redactedAttributes(context)); + // Attributes that were not marked private are inlined here, same as anywhere else. + assertEquals(LDValue.of("Oakland"), context.get("address").get("city")); + assertFalse("summary should not contain the email value anywhere", + summary.toJsonString().contains("sandy@example.com")); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void redactionAppliesToEveryKindOfAMultiContext() throws Exception { + LDContext multi = LDContext.createMulti( + LDContext.builder(ContextKind.of("user"), "user-key") + .set("email", "sandy@example.com") + .build(), + LDContext.builder(ContextKind.of("org"), "org-key") + .set("email", "billing@example.com") + .build()); + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).privateAttributes("email"), true); + try { + eventProcessor.recordIdentifyEvent(multi); + + LDValue delivered = requireEventOfKind(flushAndCollect(eventProcessor, server), + "identify").get("context"); + + assertEquals(LDValue.of("multi"), delivered.get("kind")); + assertEquals(LDValue.ofNull(), delivered.get("user").get("email")); + assertEquals(LDValue.ofNull(), delivered.get("org").get("email")); + assertEquals(redacted("email"), redactedAttributes(delivered.get("user"))); + assertEquals(redacted("email"), redactedAttributes(delivered.get("org"))); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void noPayloadContainsAPrivateValueAnywhere() throws Exception { + // A belt-and-braces check over the whole request body rather than one field, so that a + // private value cannot slip through in some event kind these tests do not name. + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, + eventsBuilder(DEFAULT_CAPACITY).privateAttributes("email"), true); + try { + eventProcessor.recordIdentifyEvent(PERSON); + eventProcessor.recordCustomEvent(PERSON, "an-event", LDValue.ofNull(), null); + eventProcessor.recordEvaluationEvent(PERSON, FLAG_KEY, 10, 1, LDValue.of(true), + null, LDValue.of(false), true, null); + + List events = flushAndCollect(eventProcessor, server); + + assertTrue("expected identify, custom, feature and summary events", + events.size() >= 4); + for (LDValue event : events) { + assertFalse("private value leaked into " + event.get("kind") + ": " + event, + event.toJsonString().contains("sandy@example.com")); + } + } finally { + eventProcessor.close(); + } + } + } + + private static List redacted(String... names) { + List result = new ArrayList<>(); + for (String name : names) { + result.add(name); + } + return result; + } + + /** The names the payload itself reports as redacted, sorted so the comparison is stable. */ + private static List redactedAttributes(LDValue context) { + List names = new ArrayList<>(); + for (LDValue name : context.get("_meta").get("redactedAttributes").values()) { + names.add(name.stringValue()); + } + java.util.Collections.sort(names); + return names; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorPrivacyWithoutPersistenceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorPrivacyWithoutPersistenceTest.java new file mode 100644 index 000000000..8d91eeecf --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorPrivacyWithoutPersistenceTest.java @@ -0,0 +1,11 @@ +package com.launchdarkly.sdk.android; + +import com.launchdarkly.sdk.android.integrations.EventPersistence; + +/** {@link EventProcessorPrivacyTest} with persistence off, which is what an application gets by default. */ +public class EventProcessorPrivacyWithoutPersistenceTest extends EventProcessorPrivacyTest { + @Override + protected EventPersistence persistence() { + return EventPersistence.DISABLED; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorTestBase.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorTestBase.java new file mode 100644 index 000000000..e4371d1de --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorTestBase.java @@ -0,0 +1,189 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.LDConfig.Builder.AutoEnvAttributes; +import com.launchdarkly.sdk.android.env.EnvironmentReporterBuilder; +import com.launchdarkly.sdk.android.env.IEnvironmentReporter; +import com.launchdarkly.sdk.android.integrations.EventPersistence; +import com.launchdarkly.sdk.android.integrations.EventProcessorBuilder; +import com.launchdarkly.sdk.android.subsystems.ClientContext; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; +import com.launchdarkly.testhelpers.httptest.Handlers; +import com.launchdarkly.testhelpers.httptest.HttpServer; +import com.launchdarkly.testhelpers.httptest.RequestInfo; + +import org.junit.Assume; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.Timeout; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Fixture for tests that drive the SDK's event processor directly and read back the analytics + * payload it posts. + */ +public abstract class EventProcessorTestBase { + protected static final String MOBILE_KEY = "test-mobile-key"; + protected static final LDContext CONTEXT = LDContext.create("user-key"); + + // Long enough that the only payload in a test is the one it asks for explicitly. + protected static final int NO_PERIODIC_FLUSH_MILLIS = 600_000; + + @Rule + public Timeout globalTimeout = Timeout.seconds(60); + @Rule + public LogCaptureRule logging = new LogCaptureRule(); + /** + * A directory of its own per test, so that the events one test persists are never found by the next. + */ + @Rule + public TemporaryFolder eventsDirectory = new TemporaryFolder(); + + private final IEnvironmentReporter environmentReporter = new EnvironmentReporterBuilder().build(); + + /** + * The processor persists events, so it needs somewhere to put them. Tests get a real directory + * rather than a stub, which is what makes them exercise the same path an application does. + */ + protected MockPlatformState platformState() { + MockPlatformState platformState = new MockPlatformState(); + platformState.setNoBackupFilesDir(eventsDirectory.getRoot()); + return platformState; + } + + protected HttpServer startEventsServer() { + return HttpServer.start(Handlers.status(202)); + } + + protected EventProcessor makeEventProcessor(HttpServer server, int capacity) { + return makeEventProcessor(server, capacity, true); + } + + protected EventProcessor makeEventProcessor(HttpServer server, int capacity, + boolean diagnosticOptOut) { + return makeEventProcessor(server, eventsBuilder(capacity), diagnosticOptOut); + } + + /** + * @return an events builder with the capacity set and periodic flushing effectively off, for + * tests that need to configure something else on top such as private attributes + */ + protected EventProcessorBuilder eventsBuilder(int capacity) { + return Components.sendEvents() + .capacity(capacity) + .eventPersistence(persistence()) + .flushIntervalMillis(NO_PERIODIC_FLUSH_MILLIS); + } + + /** + * The persistence {@link #eventsBuilder} configures. IMMEDIATE here, which is what lets a test + * assert on the store straight after a track; each suite also runs as a subclass that answers + * DISABLED, because that is what an application gets unless it asks. + */ + protected EventPersistence persistence() { + return EventPersistence.IMMEDIATE; + } + + /** Skips a test whose subject is the disk, in the subclasses that run a suite without one. */ + protected void assumePersisting() { + Assume.assumeTrue("needs a disk", persistence() != EventPersistence.DISABLED); + } + + protected EventProcessor makeEventProcessor(HttpServer server, EventProcessorBuilder events, + boolean diagnosticOptOut) { + EventProcessor eventProcessor = buildOfflineEventProcessor(server, events, diagnosticOptOut); + // LDClient turns the processor on once initialization decides the SDK is not in offline mode. + eventProcessor.setOffline(false); + return eventProcessor; + } + + /** @return the processor as the SDK builds it, before initialization has turned it on */ + protected EventProcessor buildOfflineEventProcessor(HttpServer server, EventProcessorBuilder events, + boolean diagnosticOptOut) { + return buildOfflineEventProcessor(server, events, diagnosticOptOut, platformState()); + } + + protected EventProcessor buildOfflineEventProcessor(HttpServer server, EventProcessorBuilder events, + boolean diagnosticOptOut, + PlatformState platformState) { + LDConfig config = new LDConfig.Builder(AutoEnvAttributes.Disabled) + .mobileKey(MOBILE_KEY) + .diagnosticOptOut(diagnosticOptOut) + .events(events) + .serviceEndpoints(Components.serviceEndpoints().events(server.getUri())) + .build(); + ClientContext clientContext = ClientContextImpl.fromConfig(config, MOBILE_KEY, "", + null, null, CONTEXT, logging.logger, platformState, environmentReporter, null); + return config.events.build(clientContext); + } + + /** + * Flushes and returns every event the processor posted. {@code blockingFlush} does not return + * until delivery completes, so there is no settling delay here. + */ + protected List flushAndCollect(EventProcessor eventProcessor, HttpServer server) { + eventProcessor.blockingFlush(); + return collectDelivered(server); + } + + /** Returns every event posted so far, without asking for a flush. */ + protected List collectDelivered(HttpServer server) { + List events = new ArrayList<>(); + collectRequest(server.getRecorder().requireRequest(10, TimeUnit.SECONDS), events); + while (server.getRecorder().count() > 0) { + collectRequest(server.getRecorder().requireRequest(10, TimeUnit.SECONDS), events); + } + return events; + } + + private void collectRequest(RequestInfo request, List events) { + assertEquals("POST", request.getMethod()); + assertEquals("/mobile/events/bulk", request.getPath()); + for (LDValue event : LDValue.parse(request.getBody()).values()) { + events.add(event); + } + } + + protected int countEventsOfKind(List events, String kind) { + int count = 0; + for (LDValue event : events) { + if (kind.equals(event.get("kind").stringValue())) { + count++; + } + } + return count; + } + + protected LDValue requireEventOfKind(List events, String kind) { + for (LDValue event : events) { + if (kind.equals(event.get("kind").stringValue())) { + return event; + } + } + throw new AssertionError("no event of kind " + kind + " in " + events); + } + + protected int summaryCountFor(List events, String flagKey) { + int total = 0; + boolean sawSummary = false; + for (LDValue event : events) { + if (!"summary".equals(event.get("kind").stringValue())) { + continue; + } + sawSummary = true; + // Per-context summarization means a flag can have more than one counter entry. + for (LDValue counter : event.get("features").get(flagKey).get("counters").values()) { + total += counter.get("count").intValue(); + } + } + assertTrue("no summary event was delivered", sawSummary); + return total; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorWithoutPersistenceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorWithoutPersistenceTest.java new file mode 100644 index 000000000..4d85f7e40 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventProcessorWithoutPersistenceTest.java @@ -0,0 +1,287 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.integrations.EventPersistence; +import com.launchdarkly.sdk.android.subsystems.EventProcessor; +import com.launchdarkly.testhelpers.httptest.Handler; +import com.launchdarkly.testhelpers.httptest.HttpServer; +import com.launchdarkly.testhelpers.httptest.RequestContext; + +import org.junit.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.IntUnaryOperator; + +/** + * What an application gets by default: events held in memory and delivered from there, with nothing + * on the disk. These are the situations where holding events is hardest, which the suites shared with + * persistence (run here through their WithoutPersistence subclasses) do not set out to reach. + */ +public class EventProcessorWithoutPersistenceTest extends EventProcessorTestBase { + private static final String PAYLOAD_ID_HEADER = "X-LaunchDarkly-Payload-ID"; + + @Override + protected EventPersistence persistence() { + return EventPersistence.DISABLED; + } + + /** An events service whose answer to each post can change during a test, and which keeps what it took. */ + private static final class EventsService implements Handler { + final AtomicInteger posts = new AtomicInteger(); + /** The status for the nth post, counting from 1. */ + volatile IntUnaryOperator statusForPost = post -> 202; + /** Custom event keys of each post the service accepted, by payload ID, in the order accepted. */ + final List>> accepted = + Collections.synchronizedList(new ArrayList>>()); + + @Override + public void apply(RequestContext context) { + int status = statusForPost.applyAsInt(posts.incrementAndGet()); + if (status == 202) { + List keys = new ArrayList<>(); + for (LDValue event : LDValue.parse(context.getRequest().getBody()).values()) { + if ("custom".equals(event.get("kind").stringValue())) { + keys.add(event.get("key").stringValue()); + } + } + accepted.add(new HashMap.SimpleEntry<>(context.getRequest().getHeader(PAYLOAD_ID_HEADER), keys)); + } + context.setStatus(status); + } + + /** Every accepted key once per payload ID, as the service counts them, in the order accepted. */ + List acceptedKeys() { + Set payloads = new HashSet<>(); + List keys = new ArrayList<>(); + synchronized (accepted) { + for (Map.Entry> post : accepted) { + if (payloads.add(post.getKey())) { + keys.addAll(post.getValue()); + } + } + } + return keys; + } + } + + private static boolean awaitFlush(EventProcessor eventProcessor) throws Exception { + try { + return Boolean.TRUE.equals(eventProcessor.flushAsync().get(10, TimeUnit.SECONDS)); + } catch (TimeoutException e) { + return false; + } + } + + private static List keys(String prefix, int from, int to) { + List keys = new ArrayList<>(); + for (int i = from; i < to; i++) { + keys.add(prefix + i); + } + return keys; + } + + private static void track(EventProcessor eventProcessor, String key) { + eventProcessor.recordCustomEvent(CONTEXT, key, LDValue.ofNull(), null); + } + + @Test + public void aSessionOfRefusalsRetriesAndAShutdownNeverTouchesTheFilesystem() throws Exception { + AtomicInteger asked = new AtomicInteger(); + MockPlatformState platformState = new MockPlatformState() { + @Override + public File getNoBackupFilesDir() { + asked.incrementAndGet(); + return eventsDirectory.getRoot(); + } + + @Override + public String getProcessName() { + asked.incrementAndGet(); + return "test"; + } + }; + EventsService service = new EventsService(); + // Refused and refused again on the retry; then the batch held from that and the next one are + // accepted; then the service says the key is no good. + service.statusForPost = post -> post <= 2 ? 503 : post <= 4 ? 202 : 401; + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = buildOfflineEventProcessor(server, eventsBuilder(100), + true, platformState); + eventProcessor.setOffline(false); + try { + track(eventProcessor, "refused"); + assertFalse(awaitFlush(eventProcessor)); + track(eventProcessor, "accepted"); + assertTrue(awaitFlush(eventProcessor)); + track(eventProcessor, "shut-down"); + awaitFlush(eventProcessor); + track(eventProcessor, "after-shut-down"); + } finally { + eventProcessor.close(); + } + + assertEquals(Arrays.asList("refused", "accepted"), service.acceptedKeys()); + assertEquals("posted after the service said the key is no good", 5, service.posts.get()); + } + assertEquals("a store without persistence looked for its directory", 0, asked.get()); + String[] written = eventsDirectory.getRoot().list(); + assertEquals("wrote to the disk", 0, written == null ? 0 : written.length); + } + + @Test + public void anOutageKeepsTheOldestEventsUpToCapacityAndDeliversEachOnceInOrder() throws Exception { + // Before persistence, a batch refused twice was dropped. Now it waits in memory, still counted + // against capacity, so an outage costs the newest events rather than the oldest and memory stays + // bounded however long it lasts. + int capacity = 20; + EventsService service = new EventsService(); + AtomicBoolean down = new AtomicBoolean(true); + service.statusForPost = post -> down.get() ? 503 : 202; + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = makeEventProcessor(server, eventsBuilder(capacity), true); + try { + for (String key : keys("e-", 0, 10)) { + track(eventProcessor, key); + } + assertFalse("delivered while the service was down", awaitFlush(eventProcessor)); + for (String key : keys("e-", 10, 100)) { + track(eventProcessor, key); + } + assertFalse("delivered while the service was down", awaitFlush(eventProcessor)); + + down.set(false); + assertTrue(awaitFlush(eventProcessor)); + + assertEquals(keys("e-", 0, capacity), service.acceptedKeys()); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void eventsRecordedAndFlushedFromManyThreadsThroughRefusalsAreEachDeliveredOnce() throws Exception { + int recorders = 4; + int eventsEach = 150; + EventsService service = new EventsService(); + // Every third post refused, so retries, in-memory batches waiting for the next delivery and + // flushes overlapping one another all happen while the recorders are still going. + service.statusForPost = post -> post % 3 == 0 ? 503 : 202; + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = makeEventProcessor(server, eventsBuilder(100_000), true); + try { + CountDownLatch start = new CountDownLatch(1); + CountDownLatch recorded = new CountDownLatch(recorders); + for (int r = 0; r < recorders; r++) { + String prefix = "r" + r + "-"; + new Thread(() -> { + try { + start.await(); + for (String key : keys(prefix, 0, eventsEach)) { + track(eventProcessor, key); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + recorded.countDown(); + } + }).start(); + } + Thread flusher = new Thread(() -> { + while (recorded.getCount() > 0) { + eventProcessor.flush(); + try { + Thread.sleep(2); + } catch (InterruptedException e) { + return; + } + } + }); + flusher.start(); + start.countDown(); + assertTrue(recorded.await(20, TimeUnit.SECONDS)); + flusher.join(5_000); + + service.statusForPost = post -> 202; + assertTrue("what was held back was never delivered", awaitFlush(eventProcessor)); + + List delivered = service.acceptedKeys(); + Set distinct = new HashSet<>(delivered); + assertEquals("an event arrived under two payload IDs, so the service counts it twice", + delivered.size(), distinct.size()); + Set expected = new HashSet<>(); + for (int r = 0; r < recorders; r++) { + expected.addAll(keys("r" + r + "-", 0, eventsEach)); + } + assertEquals(expected, distinct); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void eventsRecordedBeforeTheProcessorIsTurnedOnAreDeliveredOnceItIs() throws Exception { + EventsService service = new EventsService(); + try (HttpServer server = HttpServer.start(service)) { + EventProcessor eventProcessor = buildOfflineEventProcessor(server, eventsBuilder(100), true); + try { + for (String key : keys("offline-", 0, 3)) { + track(eventProcessor, key); + } + eventProcessor.flush(); + + eventProcessor.setOffline(false); + assertTrue(awaitFlush(eventProcessor)); + + assertEquals(keys("offline-", 0, 3), service.acceptedKeys()); + assertEquals("posted while offline, or posted twice", 1, service.posts.get()); + } finally { + eventProcessor.close(); + } + } + } + + @Test + public void atMostCapacityContextsAreSummarizedBetweenDeliveries() throws Exception { + // Without a disk, a commit point does not drain the summarizer, so between two deliveries it + // counts evaluations for at most capacity contexts and turns the rest away. This pins that + // trade-off: before the bound, offline evaluation across many contexts grew memory without + // limit; with persistence on, each commit point makes room again. + int capacity = 10; + try (HttpServer server = startEventsServer()) { + EventProcessor eventProcessor = makeEventProcessor(server, eventsBuilder(capacity), true); + try { + for (int round = 0; round < 2; round++) { + for (int i = 0; i < capacity * 3; i++) { + LDContext context = LDContext.create("round-" + round + "-context-" + i); + eventProcessor.recordEvaluationEvent(context, "flag", 1, 0, LDValue.of(true), null, + LDValue.of(false), false, null); + } + List events = flushAndCollect(eventProcessor, server); + assertEquals("round " + round, capacity, countEventsOfKind(events, "summary")); + } + } finally { + eventProcessor.close(); + } + } + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventStoreMultiProcessTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventStoreMultiProcessTest.java new file mode 100644 index 000000000..02b368797 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/EventStoreMultiProcessTest.java @@ -0,0 +1,839 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.LDValue; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.Timeout; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * The event store's behavior when an application runs the SDK in more than one process. + *

+ * An Android application may declare components in another process, and each process gets its own copy of + * the SDK sharing nothing but the filesystem. From the store's point of view a second process is simply a + * second instance over the same directory under a different process name, which is what these tests set + * up. That is not the same thing as a second OS process - it cannot reproduce a genuinely concurrent + * {@code O_APPEND} write from separate address spaces, which needs an instrumented test - but it does + * exercise every decision the design makes about sharing: which files a process may touch, which it must + * leave alone, and what happens when both reach for the same batch. + */ +public class EventStoreMultiProcessTest { + private static final String MAIN_PROCESS = "com.example.app"; + private static final String SERVICE_PROCESS = "com.example.app:service"; + private static final int CAPACITY = 100; + + @Rule + public Timeout globalTimeout = Timeout.seconds(30); + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + private final LDLogger logger = LDLogger.withAdapter(Logs.none(), ""); + + private File sharedDirectory() { + return new File(tempFolder.getRoot(), "events"); + } + + private EventStore storeFor(String processName) { + return storeFor(processName, CAPACITY); + } + + private EventStore storeFor(String processName, int capacity) { + return new EventStore(sharedDirectory(), processName, capacity, true, logger); + } + + private byte[] event(String key) { + return ("{\"kind\":\"custom\",\"key\":\"" + key + "\"}").getBytes(Charset.forName("UTF-8")); + } + + /** The event keys a store can currently see, which is its own log plus every shared batch. */ + private List visibleKeys(EventStore store) { + List keys = new ArrayList<>(); + for (byte[] payload : store.pendingEventPayloads()) { + keys.add(LDValue.parse(new String(payload, Charset.forName("UTF-8"))).get("key").stringValue()); + } + return keys; + } + + private void overwrite(File file, byte[] bytes) throws IOException { + try (FileOutputStream out = new FileOutputStream(file, false)) { + out.write(bytes); + } + } + + private List filesNamed(String prefix) { + List matching = new ArrayList<>(); + File[] files = sharedDirectory().listFiles(); + if (files != null) { + for (File file : files) { + if (file.getName().startsWith(prefix)) { + matching.add(file); + } + } + } + return matching; + } + + // MARK: each process writes only its own log + + @Test + public void eachProcessAppendsToItsOwnLog() { + EventStore main = storeFor(MAIN_PROCESS); + EventStore service = storeFor(SERVICE_PROCESS); + + main.stage(event("main-1")); + main.commit(); + service.stage(event("service-1")); + service.commit(); + main.stage(event("main-2")); + main.commit(); + + // Two logs, not one. This is the property that makes a spliced frame impossible: only one + // process ever has the file open for writing. + assertEquals(2, filesNamed("open-").size()); + assertEquals(Arrays.asList("main-1", "main-2"), visibleKeys(main)); + assertEquals(Arrays.asList("service-1"), visibleKeys(service)); + } + + @Test + public void processNamesThatLookAlikeInAFilenameStillGetSeparateLogs() { + // Both are legal Android process names, and a naive sanitization maps both to "com_app_remote". + EventStore colon = storeFor("com.app:remote"); + EventStore dot = storeFor("com.app.remote"); + + colon.stage(event("colon")); + colon.commit(); + dot.stage(event("dot")); + dot.commit(); + + assertNotEquals(DiskEventLog.nameFor("com.app:remote"), DiskEventLog.nameFor("com.app.remote")); + assertEquals(2, filesNamed("open-").size()); + assertEquals(Arrays.asList("colon"), visibleKeys(colon)); + assertEquals(Arrays.asList("dot"), visibleKeys(dot)); + } + + @Test + public void aLogNameIsStableAcrossRunsOfTheSameProcess() { + // Stability is what lets a process find the log its previous run left behind, so it must not + // include anything that changes from run to run. + assertEquals(DiskEventLog.nameFor(MAIN_PROCESS), DiskEventLog.nameFor(MAIN_PROCESS)); + } + + @Test + public void aProcessNameThePlatformCouldNotDetermineStillYieldsAName() { + assertTrue(DiskEventLog.nameFor("").length() > 0); + assertTrue(DiskEventLog.nameFor(null).length() > 0); + assertNotEquals(DiskEventLog.nameFor(null), DiskEventLog.nameFor(MAIN_PROCESS)); + } + + // MARK: recovery respects the boundary between processes + + @Test + public void recoveryLeavesAnotherProcessOpenLogAlone() { + EventStore service = storeFor(SERVICE_PROCESS); + service.stage(event("service-1")); + service.commit(); + + // The main process starts while the service process is still alive and still appending. Its + // recovery must not take the log out from under it. + EventStore main = storeFor(MAIN_PROCESS); + main.recoverInterruptedLog(); + + assertEquals("the other process's log was taken away", 1, + filesNamed("open-" + DiskEventLog.nameFor(SERVICE_PROCESS)).size()); + assertTrue("another process's events were claimed as recovered", main.pendingBatches().isEmpty()); + + // And the service process can still append to it and read back both events. + service.stage(event("service-2")); + service.commit(); + assertEquals(Arrays.asList("service-1", "service-2"), visibleKeys(service)); + } + + @Test + public void aProcessRecoversTheLogItsOwnPreviousRunLeftBehind() { + EventStore firstRun = storeFor(SERVICE_PROCESS); + firstRun.stage(event("before-the-crash")); + firstRun.commit(); + // No close() - the process died without getting the chance. + + EventStore secondRun = storeFor(SERVICE_PROCESS); + secondRun.recoverInterruptedLog(); + + List recovered = secondRun.pendingBatches(); + assertEquals(1, recovered.size()); + assertEquals(1, recovered.get(0).eventCount); + assertEquals(Arrays.asList("before-the-crash"), visibleKeys(secondRun)); + assertTrue("the recovered log should no longer be open", + filesNamed("open-" + DiskEventLog.nameFor(SERVICE_PROCESS)).isEmpty()); + } + + @Test + public void aCommitBeforeStartupRecoveryDoesNotPassThisRunsEventsOffAsRecovered() { + // With IMMEDIATE the first commit happens on the caller's thread, and can beat the recovery the + // processor queues on its own thread at startup. The log has the same name in both runs. + EventStore firstRun = storeFor(SERVICE_PROCESS); + firstRun.stage(event("previous-run")); + firstRun.commit(); + + EventStore secondRun = storeFor(SERVICE_PROCESS); + secondRun.stage(event("this-run")); + secondRun.commit(); + secondRun.recoverInterruptedLog(); + + List recovered = secondRun.pendingBatches(); + assertEquals(1, recovered.size()); + assertEquals("this run's event was counted as a previous run's", 1, recovered.get(0).eventCount); + assertEquals(1, filesNamed("open-" + DiskEventLog.nameFor(SERVICE_PROCESS)).size()); + assertEquals(Arrays.asList("previous-run", "this-run"), visibleKeys(secondRun)); + } + + @Test + public void aStoreWithoutPersistenceLeavesWhatAPersistentRunLeftBehindAlone() { + EventStore persistentRun = storeFor(SERVICE_PROCESS); + persistentRun.stage(event("closed-batch")); + closeOne(persistentRun); + persistentRun.stage(event("open-log")); + persistentRun.commit(); + + EventStore withoutPersistence = + new EventStore(sharedDirectory(), SERVICE_PROCESS, CAPACITY, false, logger); + withoutPersistence.recoverInterruptedLog(); + assertTrue("a store without persistence picked up a previous run's events", + withoutPersistence.pendingBatches().isEmpty()); + assertTrue(visibleKeys(withoutPersistence).isEmpty()); + + withoutPersistence.stage(event("in-memory")); + EventStore.Batch own = closeOne(withoutPersistence); + assertNotNull(own); + assertEquals(Arrays.asList(own), withoutPersistence.pendingBatches()); + + assertEquals(1, filesNamed("ready-").size()); + assertEquals(1, filesNamed("open-" + DiskEventLog.nameFor(SERVICE_PROCESS)).size()); + + // Still there for the next run that has persistence turned on. + EventStore persistentAgain = storeFor(SERVICE_PROCESS); + persistentAgain.recoverInterruptedLog(); + assertEquals(2, persistentAgain.pendingBatches().size()); + } + + @Test + public void aStoreWithoutPersistenceAsksTheFilesystemNothingEvenAboutABatchItDoesNotHold() + throws IOException { + AtomicInteger reads = new AtomicInteger(); + EventStore store = new EventStore(sharedDirectory(), MAIN_PROCESS, CAPACITY, false, logger) { + @Override + InputStream openForReading(File file) throws IOException { + reads.incrementAndGet(); + return super.openForReading(file); + } + }; + + store.stage(event("held")); + EventStore.Batch held = closeOne(store); + assertNotNull(held); + assertEquals(Arrays.asList(held), store.pendingBatches()); + assertEquals(LDValue.of("held"), + LDValue.parse(new String(store.body(held), Charset.forName("UTF-8"))).get(0).get("key")); + store.remove(held); + + // A batch this store never closed, which is what sends the paths that form a file name down the + // branch that would otherwise ask the filesystem about it. + EventStore.Batch stranger = new EventStore.Batch(UUID.randomUUID().toString(), 1); + assertNull(store.body(stranger)); + store.remove(stranger); + + assertEquals(0, reads.get()); + assertFalse("a store without persistence created its directory", sharedDirectory().exists()); + } + + @Test + public void oneProcessRecoveringDoesNotDisturbWhatAnotherHasAlreadyClosed() { + EventStore service = storeFor(SERVICE_PROCESS); + service.stage(event("service-1")); + EventStore.Batch fromService = closeOne(service); + assertNotNull(fromService); + + // Recovery runs when a store is constructed, before that process has recorded anything of its own, + // which is the only time it is correct to go looking for a log to close. + EventStore main = storeFor(MAIN_PROCESS); + main.recoverInterruptedLog(); + main.stage(event("main-1")); + main.commit(); + + // The service process's batch is untouched and still deliverable, by either of them. + assertEquals(1, filesNamed("ready-").size()); + assertTrue(main.pendingBatches().contains(fromService)); + } + + // MARK: closed batches are shared on purpose + + @Test + public void aBatchClosedByOneProcessIsDeliverableByAnother() throws IOException { + // The point of sharing: a service process that records events and is then killed may never run + // again, so a batch only it could see would never be delivered. + EventStore service = storeFor(SERVICE_PROCESS); + service.stage(event("recorded-by-the-service")); + EventStore.Batch batch = closeOne(service); + assertNotNull(batch); + + EventStore main = storeFor(MAIN_PROCESS); + List pending = main.pendingBatches(); + + assertEquals(1, pending.size()); + EventStore.Batch seenByMain = pending.get(0); + // The same payload ID, which is what keeps two processes delivering this batch from counting the + // events twice upstream. + assertEquals(batch.payloadId, seenByMain.payloadId); + assertEquals(batch.eventCount, seenByMain.eventCount); + + byte[] body = main.body(seenByMain); + assertNotNull("the other process could not read the batch", body); + LDValue parsed = LDValue.parse(new String(body, Charset.forName("UTF-8"))); + assertEquals(1, parsed.size()); + assertEquals(LDValue.of("recorded-by-the-service"), parsed.get(0).get("key")); + } + + @Test + public void aClosedBatchIsCountedFromItsNameWithoutReadingItBack() throws IOException { + EventStore main = storeFor(MAIN_PROCESS); + main.stage(event("first")); + main.stage(event("second")); + EventStore.Batch batch = closeOne(main); + assertNotNull(batch); + + // Nothing appends to a batch once it is closed, so the count its name carries still holds and + // listing it has no reason to go to the disk. Damaging the file is how the test tells the two + // apart: a listing that read would refuse this batch. + List ready = filesNamed("ready-"); + assertEquals(1, ready.size()); + assertEquals("ready-" + batch.payloadId + "-2", ready.get(0).getName()); + overwrite(ready.get(0), "not an event log".getBytes(Charset.forName("UTF-8"))); + + // The process that closed it and one that did not both count it from the name alone. + for (EventStore store : Arrays.asList(main, storeFor(SERVICE_PROCESS))) { + List pending = store.pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(batch.payloadId, pending.get(0).payloadId); + assertEquals(2, pending.get(0).eventCount); + } + + // The damage is found where the batch is read to be sent, and reads as a loss rather than as a + // failure to retry, so the delivery removes it instead of trying it forever, and says it lost events. + try { + main.body(batch); + fail("a damaged batch read as something to send or as nothing to lose"); + } catch (EventStore.UnreadableBatchException expected) { + // what a delivery takes as lost + } + } + + /** + * Renames a batch to the name a version that did not count them in the name gave it, which is what + * such a version leaves behind for this one to deliver. + */ + private File asUncountedName(File batch) { + String name = batch.getName(); + File renamed = new File(batch.getParentFile(), name.substring(0, name.lastIndexOf('-'))); + assertTrue("could not rename " + name, batch.renameTo(renamed)); + return renamed; + } + + @Test + public void aBatchNamedWithoutACountIsReadOnceAndThenRemembered() throws IOException { + EventStore previousVersion = storeFor(SERVICE_PROCESS); + previousVersion.stage(event("left-by-a-previous-version")); + closeOne(previousVersion); + assertEquals(1, filesNamed("ready-").size()); + asUncountedName(filesNamed("ready-").get(0)); + + AtomicInteger reads = new AtomicInteger(); + EventStore main = new EventStore(sharedDirectory(), MAIN_PROCESS, CAPACITY, true, logger) { + @Override + InputStream openForReading(File file) throws IOException { + reads.incrementAndGet(); + return super.openForReading(file); + } + }; + + List pending = main.pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(1, pending.get(0).eventCount); + assertEquals("a batch with no count in its name was not read", 1, reads.get()); + + // Counted once and remembered, rather than read again by every listing for the rest of the session. + assertEquals(pending, main.pendingBatches()); + assertEquals(1, reads.get()); + + // And still deliverable under the name it has, which is not the one its count would give it. + assertNotNull(main.body(pending.get(0))); + main.remove(pending.get(0)); + assertTrue(filesNamed("ready-").isEmpty()); + } + + @Test + public void aPayloadIdEndingInZeroLedDigitsIsNotTakenForACount() { + EventStore previousVersion = storeFor(SERVICE_PROCESS); + previousVersion.stage(event("left-by-a-previous-version")); + closeOne(previousVersion); + File batch = filesNamed("ready-").get(0); + // Its last group fits an int once the leading zeros are dropped. + String payloadId = "6F9619FF-8B86-D011-B42D-000012345678"; + assertTrue(batch.renameTo(new File(batch.getParentFile(), "ready-" + payloadId))); + + List pending = storeFor(MAIN_PROCESS).pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(payloadId, pending.get(0).payloadId); + assertEquals(1, pending.get(0).eventCount); + } + + @Test + public void aBatchTheOtherProcessAlreadyDeliveredReadsAsNothingToSend() throws IOException { + EventStore service = storeFor(SERVICE_PROCESS); + service.stage(event("delivered-once")); + EventStore.Batch batch = closeOne(service); + assertNotNull(batch); + + EventStore main = storeFor(MAIN_PROCESS); + main.remove(batch); // main delivered it first + + // The other process asking for the body gets nothing, which its caller treats as already + // delivered rather than as a failure to retry. + assertNull(service.body(batch)); + } + + @Test + public void bothProcessesRemovingTheSameBatchIsHarmless() { + EventStore service = storeFor(SERVICE_PROCESS); + service.stage(event("delivered-twice")); + EventStore.Batch batch = closeOne(service); + assertNotNull(batch); + + EventStore main = storeFor(MAIN_PROCESS); + main.remove(batch); + service.remove(batch); // a remove of a file that is already gone + + assertTrue(filesNamed("ready-").isEmpty()); + assertEquals(0, main.pendingBatches().size()); + assertEquals(0, service.pendingBatches().size()); + } + + @Test + public void aProcessCountsAnotherProcessPendingBatchesAgainstItsOwnCapacity() { + // Deliberately conservative, and documented as such: the alternative is cross-process accounting, + // which would need cross-process locking to be correct. + EventStore service = storeFor(SERVICE_PROCESS); + for (int i = 0; i < 3; i++) { + service.stage(event("service-" + i)); + } + closeOne(service); + + EventStore main = storeFor(MAIN_PROCESS); + main.pendingBatches(); // this is what reconciles the count with the directory + + assertEquals(3, main.getPendingEventCount()); + } + + // MARK: a read that fails is no reason to delete + + /** + * A store whose reads fail while {@code failing} is set, the way they do in a process that has run + * out of file descriptors: the files are intact and readable again once some are freed. + */ + private EventStore storeWithFailingReads(String processName, AtomicBoolean failing) { + return new EventStore(sharedDirectory(), processName, CAPACITY, true, logger) { + @Override + InputStream openForReading(File file) throws IOException { + if (failing.get()) { + throw new FileNotFoundException(file + " (Too many open files)"); + } + return super.openForReading(file); + } + }; + } + + @Test + public void aBatchNamedWithoutACountThatCannotBeReadIsLeftForALaterListing() { + EventStore service = storeFor(SERVICE_PROCESS); + service.stage(event("recorded-by-the-service")); + closeOne(service); + asUncountedName(filesNamed("ready-").get(0)); + + AtomicBoolean failing = new AtomicBoolean(true); + EventStore main = storeWithFailingReads(MAIN_PROCESS, failing); + assertTrue(main.pendingBatches().isEmpty()); + assertEquals("a batch that could not be read was deleted", 1, filesNamed("ready-").size()); + + failing.set(false); + List pending = main.pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(1, pending.get(0).eventCount); + } + + @Test + public void aBatchCountedByItsNameIsListedEvenWhileReadsAreFailing() { + EventStore service = storeFor(SERVICE_PROCESS); + service.stage(event("recorded-by-the-service")); + closeOne(service); + + // The read a listing used to need is the one a process out of file descriptors cannot make. With + // the count in the name there is nothing to read until the batch is sent, and a body that cannot + // be read is a delivery to retry rather than a batch that is invisible until it can be. + AtomicBoolean failing = new AtomicBoolean(true); + EventStore main = storeWithFailingReads(MAIN_PROCESS, failing); + List pending = main.pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(1, pending.get(0).eventCount); + try { + main.body(pending.get(0)); + fail("a batch that could not be read was reported as having nothing to send"); + } catch (IOException expected) { + // what the processor retries on + } + } + + @Test + public void aBodyThatCannotBeReadIsAFailureToRetryRatherThanNothingToSend() throws IOException { + AtomicBoolean failing = new AtomicBoolean(false); + EventStore main = storeWithFailingReads(MAIN_PROCESS, failing); + main.stage(event("kept")); + EventStore.Batch batch = closeOne(main); + assertNotNull(batch); + + failing.set(true); + try { + main.body(batch); + fail("a batch that could not be read was reported as having nothing to send"); + } catch (IOException expected) { + // what the processor retries on + } + assertEquals(1, filesNamed("ready-").size()); + + failing.set(false); + byte[] body = main.body(batch); + assertNotNull(body); + assertEquals(LDValue.of("kept"), + LDValue.parse(new String(body, Charset.forName("UTF-8"))).get(0).get("key")); + } + + @Test + public void aPreviousRunsLogThatCannotBeReadIsKeptForALaterListing() { + EventStore firstRun = storeFor(SERVICE_PROCESS); + firstRun.stage(event("before-the-crash")); + firstRun.commit(); + + AtomicBoolean failing = new AtomicBoolean(true); + EventStore secondRun = storeWithFailingReads(SERVICE_PROCESS, failing); + secondRun.recoverInterruptedLog(); + // Moved out from under the name this run appends to, and not deleted. + assertTrue(filesNamed("open-" + DiskEventLog.nameFor(SERVICE_PROCESS)).isEmpty()); + assertEquals(1, filesNamed("ready-").size()); + + secondRun.stage(event("this-run")); + secondRun.commit(); + failing.set(false); + List pending = secondRun.pendingBatches(); + assertEquals(1, pending.size()); + assertEquals(1, pending.get(0).eventCount); + assertEquals(Arrays.asList("before-the-crash", "this-run"), visibleKeys(secondRun)); + } + + // MARK: a write that fails partway + + /** + * A store whose writes land whole until {@code whole} of them have, after which each gets all but + * the last few bytes onto the disk and fails, as one does when the device fills mid-write. + */ + private EventStore storeFailingPartwayAfter(int whole) { + AtomicInteger writes = new AtomicInteger(); + return new EventStore(sharedDirectory(), MAIN_PROCESS, CAPACITY, true, logger) { + @Override + FileOutputStream openForAppending(File log) throws IOException { + return new FileOutputStream(log, true) { + @Override + public void write(byte[] bytes) throws IOException { + if (writes.incrementAndGet() <= whole) { + super.write(bytes); + return; + } + super.write(bytes, 0, bytes.length - 3); + throw new IOException("No space left on device"); + } + }; + } + }; + } + + @Test + public void eventsBeingWrittenStillCountAgainstCapacity() throws Exception { + // A commit takes the staged events out of the buffer and writes them outside the buffer's lock. + // They used to be counted nowhere until the write returned, and capacity admitted that many more. + CountDownLatch writing = new CountDownLatch(1); + CountDownLatch letWriteFinish = new CountDownLatch(1); + EventStore store = new EventStore(sharedDirectory(), MAIN_PROCESS, 2, true, logger) { + @Override + FileOutputStream openForAppending(File log) throws IOException { + writing.countDown(); + try { + letWriteFinish.await(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return super.openForAppending(log); + } + }; + assertTrue(store.stage(event("first"), false)); + assertTrue(store.stage(event("second"), false)); + + Thread committer = new Thread(store::commit); + committer.start(); + try { + assertTrue("the commit never reached the disk", writing.await(5, TimeUnit.SECONDS)); + + assertEquals(2, store.getPendingEventCount()); + assertFalse("admitted past capacity while the write was under way", + store.stage(event("third"), false)); + } finally { + letWriteFinish.countDown(); + committer.join(5_000); + } + + assertEquals(2, store.getPendingEventCount()); + assertFalse(store.stage(event("third"), false)); + assertEquals(Arrays.asList("first", "second"), visibleKeys(store)); + } + + @Test + public void removingABatchTwiceStopsCountingItOnce() { + // Deliveries can overlap, and two of them can both get to the same batch. + for (boolean persist : new boolean[] {true, false}) { + EventStore store = new EventStore(new File(sharedDirectory(), "persist-" + persist), + MAIN_PROCESS, CAPACITY, persist, logger); + store.stage(event("first")); + EventStore.Batch first = closeOne(store); + store.stage(event("second")); + store.closeBatch(); + store.pendingBatches(); + + store.remove(first); + store.remove(first); + + assertEquals("persisting: " + persist, 1, store.getPendingEventCount()); + } + } + + @Test + public void aDeliveredBatchWhoseFileWillNotDeleteIsNotListedAgain() { + EventStore store = storeFor(MAIN_PROCESS); + store.stage(event("first")); + EventStore.Batch first = closeOne(store); + store.stage(event("second")); + EventStore.Batch second = closeOne(store); + assertNotNull(first); + assertNotNull(second); + + File directory = sharedDirectory(); + // A directory with no write permission is one its entries cannot be unlinked from, which is what the + // filesystem refusing a delete looks like from here. + assertTrue("could not make the directory read-only", directory.setWritable(false)); + try { + store.remove(first); + + // Delivered, so it neither counts against capacity nor comes back for another delivery, even + // though its file is still sitting there. + assertEquals(Arrays.asList(second), store.pendingBatches()); + assertEquals(1, store.getPendingEventCount()); + } finally { + assertTrue("could not make the directory writable again", directory.setWritable(true)); + } + + // The listing that can delete it does, rather than leaving it for a run that would send it again. + store.pendingBatches(); + assertTrue(filesNamed("ready-" + first.payloadId).isEmpty()); + } + + @Test + public void aBatchAnotherProcessDeliveredStopsCountingOnce() { + EventStore main = storeFor(MAIN_PROCESS); + EventStore service = storeFor(SERVICE_PROCESS); + main.stage(event("first")); + EventStore.Batch first = closeOne(main); + main.stage(event("second")); + main.closeBatch(); + + // The other process delivers it, and this one lists again before its own delivery of it finishes. + service.remove(first); + main.pendingBatches(); + main.remove(first); + + assertEquals(1, main.getPendingEventCount()); + } + + @Test + public void aListingDoesNotLoseOrDoubleABatchClosedWhileItRuns() throws Exception { + for (boolean persist : new boolean[] {true, false}) { + EventStore store = new EventStore(new File(sharedDirectory(), "persist-" + persist), + MAIN_PROCESS, 100_000, persist, logger); + AtomicBoolean done = new AtomicBoolean(); + Thread lister = new Thread(() -> { + while (!done.get()) { + store.pendingBatches(); + // Without it the listings can hold the locks so continuously that the closes starve. + Thread.yield(); + } + }); + lister.start(); + try { + // Checked after every close, because the next listing would correct a miscount before + // the end of the loop could see it. + for (int i = 1; i <= 150; i++) { + store.stage(event("event-" + i)); + store.closeBatch(); + assertEquals("persisting: " + persist, i, store.getPendingEventCount()); + } + } finally { + done.set(true); + lister.join(5_000); + } + } + } + + /** Closes what a test expects to be exactly one batch, as anything short of a failed write leaves. */ + private static EventStore.Batch closeOne(EventStore store) { + List closed = store.closeBatch(); + assertEquals(1, closed.size()); + return closed.get(0); + } + + private List keysIn(byte[] body) { + List keys = new ArrayList<>(); + for (LDValue event : LDValue.parse(new String(body, Charset.forName("UTF-8"))).values()) { + keys.add(event.get("key").stringValue()); + } + return keys; + } + + @Test + public void framesAWriteLandedBeforeItFailedAreSentOnceNotFromTheLogAndMemoryBoth() + throws IOException { + // The file header and the first event land. The next write gets a whole frame and part of + // another onto the disk before the device is full. + EventStore store = storeFailingPartwayAfter(2); + + store.stage(event("written")); + store.commit(); + store.stage(event("landed")); + store.stage(event("torn")); + store.commit(); + assertFalse(store.isPersisting()); + + List closed = store.closeBatch(); + assertEquals("the log's batch and memory's, both reported", 2, closed.size()); + assertEquals(Arrays.asList("written"), keysIn(store.body(closed.get(0)))); + assertEquals(Arrays.asList("landed", "torn"), keysIn(store.body(closed.get(1)))); + + List> bodies = new ArrayList<>(); + for (EventStore.Batch batch : store.pendingBatches()) { + bodies.add(keysIn(store.body(batch))); + } + assertEquals(Arrays.asList(Arrays.asList("written"), Arrays.asList("landed", "torn")), bodies); + } + + @Test + public void theNextRunHasNothingToRecoverFromAFirstWriteThatFailedPartway() throws IOException { + // The file header lands, and then a whole frame and part of another before the device is full. + EventStore store = storeFailingPartwayAfter(1); + + store.stage(event("landed")); + store.stage(event("torn")); + store.commit(); + EventStore.Batch fromMemory = closeOne(store); + assertNotNull(fromMemory); + assertEquals(Arrays.asList("landed", "torn"), keysIn(store.body(fromMemory))); + + // Delivered from memory this run, so the next must not find any of it again in the log. + EventStore nextRun = storeFor(MAIN_PROCESS); + nextRun.recoverInterruptedLog(); + assertEquals(0, nextRun.pendingBatches().size()); + } + + // MARK: the two together + + @Test + public void concurrentRecordingFromTwoProcessesLosesAndCorruptsNothing() throws Exception { + int eventsPerProcess = 200; + // Capacity well clear of what the test records, so that a dropped event means a real loss rather + // than the store correctly refusing to exceed its limit. + EventStore main = storeFor(MAIN_PROCESS, 10 * eventsPerProcess); + EventStore service = storeFor(SERVICE_PROCESS, 10 * eventsPerProcess); + + CountDownLatch startTogether = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(2); + Thread mainThread = recordingThread(main, "main", eventsPerProcess, startTogether, finished); + Thread serviceThread = recordingThread(service, "service", eventsPerProcess, startTogether, finished); + mainThread.start(); + serviceThread.start(); + startTogether.countDown(); + assertTrue(finished.await(20, TimeUnit.SECONDS)); + + // Every event either process recorded is readable, and every frame parses - a spliced frame would + // show up as a parse failure or a missing event, because a torn frame ends the reader. + Set everything = new HashSet<>(visibleKeys(main)); + everything.addAll(visibleKeys(service)); + assertEquals(2 * eventsPerProcess, everything.size()); + for (int i = 0; i < eventsPerProcess; i++) { + assertTrue("lost main-" + i, everything.contains("main-" + i)); + assertTrue("lost service-" + i, everything.contains("service-" + i)); + } + } + + private Thread recordingThread( + EventStore store, + String prefix, + int count, + CountDownLatch startTogether, + CountDownLatch finished + ) { + return new Thread(() -> { + try { + startTogether.await(); + for (int i = 0; i < count; i++) { + store.stage(event(prefix + "-" + i)); + // Committing on every event maximizes the chance of two writes overlapping, which is + // the interleaving this is looking for. + store.commit(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + finished.countDown(); + } + }); + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/OutboundEventBufferSerializationTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/OutboundEventBufferSerializationTest.java new file mode 100644 index 000000000..e0b272ba2 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/OutboundEventBufferSerializationTest.java @@ -0,0 +1,167 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.logging.LDLogAdapter; +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.internal.events.Event; +import com.launchdarkly.sdk.internal.events.EventOutputFormatter; +import com.launchdarkly.sdk.internal.events.EventSummarizer; +import com.launchdarkly.sdk.internal.events.EventsConfiguration; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.StringWriter; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * The serialization the on-disk event log is built on: one event in, one JSON object out. + *

+ * These are deliberately narrow. Everything the log does depends on a frame holding exactly one + * event object, so the assumption is checked here rather than inferred from a delivered payload. + */ +public class OutboundEventBufferSerializationTest { + private static final LDContext CONTEXT = LDContext.create("user-key"); + private static final String FLAG_KEY = "flag-key"; + + private final LDLogAdapter logAdapter = Logs.none(); + + private OutboundEventBuffer makeBuffer() { + // Unbounded cardinality, so these stay about serialization and nothing else. + return new OutboundEventBuffer(false, Collections.emptyList(), true, Integer.MAX_VALUE, + LDLogger.withAdapter(logAdapter, "")); + } + + private LDValue parse(byte[] serialized) { + return LDValue.parse(new String(serialized, Charset.forName("UTF-8"))); + } + + @Test + public void customEventSerializesToASingleObject() { + byte[] serialized = makeBuffer().serialize( + new Event.Custom(1000, "an-event", CONTEXT, LDValue.of("data"), 2.5)); + + assertNotNull(serialized); + // The bytes are one object, with no array around it, so that a payload can be assembled by + // joining frames with commas. + assertEquals('{', serialized[0]); + assertEquals('}', serialized[serialized.length - 1]); + + LDValue event = parse(serialized); + assertEquals(LDValue.of("custom"), event.get("kind")); + assertEquals(LDValue.of("an-event"), event.get("key")); + assertEquals(LDValue.of(2.5), event.get("metricValue")); + } + + @Test + public void identifyEventSerializesToASingleObject() { + byte[] serialized = makeBuffer().serialize(new Event.Identify(1000, CONTEXT)); + + assertNotNull(serialized); + LDValue event = parse(serialized); + assertEquals(LDValue.of("identify"), event.get("kind")); + assertEquals(LDValue.of(CONTEXT.getKey()), event.get("context").get("key")); + } + + @Test + public void featureEventSerializesToASingleObject() { + Event.FeatureRequest event = new Event.FeatureRequest(1000, FLAG_KEY, CONTEXT, 10, 1, + LDValue.of(true), LDValue.of(false), null, null, true, null, false); + + byte[] serialized = makeBuffer().serialize(event); + + assertNotNull(serialized); + LDValue parsed = parse(serialized); + assertEquals(LDValue.of("feature"), parsed.get("kind")); + assertEquals(LDValue.of(FLAG_KEY), parsed.get("key")); + assertEquals(LDValue.of(1), parsed.get("variation")); + } + + @Test + public void summariesSerializeOneObjectEachAndLeaveTheCountersEmpty() { + OutboundEventBuffer buffer = makeBuffer(); + buffer.summarize(new Event.FeatureRequest(1000, FLAG_KEY, CONTEXT, 10, 1, LDValue.of(true), + LDValue.of(false), null, null, false, null, false)); + buffer.summarize(new Event.FeatureRequest(1001, FLAG_KEY, CONTEXT, 10, 1, LDValue.of(true), + LDValue.of(false), null, null, false, null, false)); + + List summaries = buffer.serializeSummariesAndReset(); + + assertEquals(1, summaries.size()); + LDValue summary = parse(summaries.get(0)); + assertEquals(LDValue.of("summary"), summary.get("kind")); + int counted = 0; + for (LDValue counter : summary.get("features").get(FLAG_KEY).get("counters").values()) { + counted += counter.get("count").intValue(); + } + assertEquals(2, counted); + + // Reset, so the same evaluations are not summarized again by the next call. + assertTrue(buffer.serializeSummariesAndReset().isEmpty()); + } + + @Test + public void theContextLimitHoldsWhateverOrderContextsArriveIn() { + OutboundEventBuffer buffer = new OutboundEventBuffer(false, Collections.emptyList(), true, 1, + LDLogger.withAdapter(logAdapter, "")); + LDContext other = LDContext.create("other-key"); + + assertTrue(summarize(buffer, CONTEXT)); + assertTrue(summarize(buffer, CONTEXT)); + // Equal to the one counted but a different instance, so it misses the reference check and + // has to be found in the set rather than turned away. + assertTrue(summarize(buffer, LDContext.create(CONTEXT.getKey()))); + assertEquals(false, summarize(buffer, other)); + assertTrue(summarize(buffer, CONTEXT)); + assertEquals(false, summarize(buffer, other)); + + buffer.takeSummaries(); + + // The limit lifts with the drain, for the context seen last as much as any other. + assertTrue(summarize(buffer, other)); + assertEquals(false, summarize(buffer, CONTEXT)); + } + + private static boolean summarize(OutboundEventBuffer buffer, LDContext context) { + return buffer.summarize(1000, FLAG_KEY, 10, 1, LDValue.of(true), LDValue.of(false), context); + } + + @Test + public void serializingSummariesWithNothingCountedReturnsNothing() { + assertTrue(makeBuffer().serializeSummariesAndReset().isEmpty()); + } + + @Test + public void framesAssembleIntoTheSamePayloadTheFormatterWritesForTheWholeRun() throws Exception { + Event[] run = { + new Event.Custom(1000, "first", CONTEXT, LDValue.ofNull(), null), + new Event.Custom(1001, "second", CONTEXT, LDValue.ofNull(), null) + }; + + ByteArrayOutputStream frames = new ByteArrayOutputStream(); + for (byte[] serialized : makeBuffer().serializeAll(Arrays.asList(run))) { + EventLogFormat.writeFrame(frames, serialized); + } + byte[] body = EventLogFormat.assembleBody(EventLogFormat.withFileHeader(frames.toByteArray())); + + // Configured as makeBuffer() configures the formatter it wraps, so any difference is the + // framing's and not the configuration's. + EventOutputFormatter formatter = new EventOutputFormatter(new EventsConfiguration(false, 0, + null, 0, null, null, 1, null, 0, false, false, Collections.emptyList(), true)); + StringWriter expected = new StringWriter(); + assertEquals(run.length, formatter.writeOutputEvents(run, + Collections.emptyList(), expected)); + + assertNotNull(body); + assertEquals(LDValue.parse(expected.toString()), parse(body)); + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/integrations/LDCrashHandlerTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/integrations/LDCrashHandlerTest.java new file mode 100644 index 000000000..97cb317ed --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/integrations/LDCrashHandlerTest.java @@ -0,0 +1,118 @@ +package com.launchdarkly.sdk.android.integrations; + +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.sdk.android.LDClientInterface; +import com.launchdarkly.sdk.android.LaunchDarklyException; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class LDCrashHandlerTest { + private final List calls = new ArrayList<>(); + private final Thread crashed = new Thread("crashed"); + private final Throwable crash = new RuntimeException("boom"); + private final Thread.UncaughtExceptionHandler next = (thread, throwable) -> { + assertSame(crashed, thread); + assertSame(crash, throwable); + calls.add("next"); + }; + + private Thread.UncaughtExceptionHandler originalDefault; + + @Before + public void saveDefaultHandler() { + originalDefault = Thread.getDefaultUncaughtExceptionHandler(); + } + + @After + public void restoreDefaultHandler() { + Thread.setDefaultUncaughtExceptionHandler(originalDefault); + } + + @Test + public void deliversTheEventsBeforePassingTheCrashOn() { + LDClientInterface client = createMock(LDClientInterface.class); + expect(client.flushAndWait(eq(2000L), eq(TimeUnit.MILLISECONDS))).andAnswer(() -> { + calls.add("flush"); + return true; + }); + replay(client); + + new LDCrashHandler(() -> client, 2000, next).uncaughtException(crashed, crash); + + verify(client); + assertEquals(Arrays.asList("flush", "next"), calls); + } + + @Test + public void passesTheCrashOnWhenTheEventsAreNotDelivered() { + LDClientInterface client = createMock(LDClientInterface.class); + expect(client.flushAndWait(eq(2000L), eq(TimeUnit.MILLISECONDS))).andReturn(false); + replay(client); + + new LDCrashHandler(() -> client, 2000, next).uncaughtException(crashed, crash); + + verify(client); + assertEquals(Arrays.asList("next"), calls); + } + + @Test + public void passesTheCrashOnWhenTheClientWasNeverInitialized() { + new LDCrashHandler(() -> { + throw new LaunchDarklyException("LDClient.get() was called before init()!"); + }, 2000, next).uncaughtException(crashed, crash); + + assertEquals(Arrays.asList("next"), calls); + } + + @Test + public void passesTheCrashOnWhenTheFlushThrows() { + LDClientInterface client = createMock(LDClientInterface.class); + expect(client.flushAndWait(eq(2000L), eq(TimeUnit.MILLISECONDS))) + .andThrow(new IllegalStateException("the crash broke the client too")); + replay(client); + + new LDCrashHandler(() -> client, 2000, next).uncaughtException(crashed, crash); + + assertEquals(Arrays.asList("next"), calls); + } + + @Test + public void installsInFrontOfTheCurrentDefaultHandler() { + Thread.setDefaultUncaughtExceptionHandler(next); + + LDCrashHandler.install(2, TimeUnit.SECONDS); + Thread.UncaughtExceptionHandler installed = Thread.getDefaultUncaughtExceptionHandler(); + installed.uncaughtException(crashed, crash); + + assertTrue(installed instanceof LDCrashHandler); + assertEquals(Arrays.asList("next"), calls); + } + + @Test + public void installingTwiceKeepsTheFirstHandler() { + Thread.setDefaultUncaughtExceptionHandler(next); + + LDCrashHandler.install(2, TimeUnit.SECONDS); + Thread.UncaughtExceptionHandler first = Thread.getDefaultUncaughtExceptionHandler(); + LDCrashHandler.install(5, TimeUnit.SECONDS); + + assertSame(first, Thread.getDefaultUncaughtExceptionHandler()); + first.uncaughtException(crashed, crash); + assertEquals(Arrays.asList("next"), calls); + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizerTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizerTest.java new file mode 100644 index 000000000..057db864f --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/internal/events/SameContextPerContextEventSummarizerTest.java @@ -0,0 +1,99 @@ +package com.launchdarkly.sdk.internal.events; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.launchdarkly.sdk.ContextKind; +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.LDValue; + +import org.junit.Test; + +import java.util.List; + +/** + * Checks the summarizer against the one it stands in for. The JVM these run on has + * {@code computeIfAbsent}, so the original can be run alongside it and the two compared directly. + */ +public class SameContextPerContextEventSummarizerTest { + private static final LDContext USER = LDContext.builder("user").name("User").set("plan", "pro").build(); + private static final LDContext DEVICE = LDContext.create(ContextKind.of("device"), "device"); + private static final LDContext MULTI = LDContext.createMulti(USER, DEVICE); + + private static void summarizeTheSame(EventSummarizerInterface a, EventSummarizerInterface b) { + Object[][] evaluations = { + {1000L, "flag-a", 1, 0, LDValue.of(true), LDValue.of(false), USER}, + {1001L, "flag-a", 1, 0, LDValue.of(true), LDValue.of(false), USER}, + {1002L, "flag-a", 2, 1, LDValue.of(false), LDValue.of(false), USER}, + {999L, "flag-b", -1, -1, LDValue.of("x"), LDValue.of("x"), DEVICE}, + {1003L, "flag-a", 1, 0, LDValue.of(true), LDValue.of(false), MULTI}, + {1004L, "flag-c", 3, 2, LDValue.of(2.5), LDValue.ofNull(), LDContext.builder("user") + .name("User").set("plan", "pro").build()}, + }; + for (Object[] e : evaluations) { + for (EventSummarizerInterface s : new EventSummarizerInterface[]{a, b}) { + s.summarizeEvent((Long) e[0], (String) e[1], (Integer) e[2], (Integer) e[3], + (LDValue) e[4], (LDValue) e[5], (LDContext) e[6]); + } + } + } + + @Test + public void summarizesAsTheOriginalDoes() { + PerContextEventSummarizer original = new PerContextEventSummarizer(); + SameContextPerContextEventSummarizer copy = new SameContextPerContextEventSummarizer(); + summarizeTheSame(original, copy); + + assertFalse(copy.isEmpty()); + List expected = original.getSummariesAndReset(); + List actual = copy.getSummariesAndReset(); + // An equal context built separately shares a summary, so three contexts, not four. + assertEquals(3, actual.size()); + assertEquals(expected, actual); + assertTrue(copy.isEmpty()); + assertTrue(copy.getSummariesAndReset().isEmpty()); + } + + @Test + public void restoresAsTheOriginalDoes() { + PerContextEventSummarizer original = new PerContextEventSummarizer(); + SameContextPerContextEventSummarizer copy = new SameContextPerContextEventSummarizer(); + summarizeTheSame(original, copy); + List taken = copy.getSummariesAndReset(); + original.restoreTo(original.getSummariesAndReset()); + copy.restoreTo(taken); + + summarizeTheSame(original, copy); + + assertEquals(original.getSummariesAndReset(), copy.getSummariesAndReset()); + } + + @Test + public void theContextSeenLastIsCountedAfreshAfterEveryReset() { + SameContextPerContextEventSummarizer copy = new SameContextPerContextEventSummarizer(); + PerContextEventSummarizer original = new PerContextEventSummarizer(); + for (Runnable reset : new Runnable[]{ + copy::getSummariesAndReset, + copy::clear, + () -> copy.restoreTo(java.util.Collections.emptyList())}) { + copy.summarizeEvent(1000, "flag", 1, 0, LDValue.of(true), LDValue.of(false), USER); + reset.run(); + copy.summarizeEvent(1001, "flag", 1, 0, LDValue.of(true), LDValue.of(false), USER); + original.summarizeEvent(1001, "flag", 1, 0, LDValue.of(true), LDValue.of(false), USER); + + assertEquals(original.getSummariesAndReset(), copy.getSummariesAndReset()); + } + } + + @Test + public void clearForgetsEverything() { + SameContextPerContextEventSummarizer copy = new SameContextPerContextEventSummarizer(); + copy.summarizeEvent(1000, "flag", 1, 0, LDValue.of(true), LDValue.of(false), USER); + + copy.clear(); + + assertTrue(copy.isEmpty()); + assertTrue(copy.getSummariesAndReset().isEmpty()); + } +} diff --git a/settings.gradle b/settings.gradle index 36c077f5b..fa5361a28 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,5 @@ include(":launchdarkly-android-client-sdk") include(":shared-test-code") include(":example") +include(":test-app") include(":contract-tests") diff --git a/shared-test-code/build.gradle b/shared-test-code/build.gradle index c3fb0e054..9bbaa5785 100644 --- a/shared-test-code/build.gradle +++ b/shared-test-code/build.gradle @@ -12,7 +12,7 @@ ext.versions = [ "androidAnnotation": "1.2.0", "gson": "2.13.2", "junit": "4.13", - "launchdarklyJavaSdkInternal": "1.9.0", + "launchdarklyJavaSdkInternal": "1.12.1", "launchdarklyLogging": "1.1.1", ] diff --git a/shared-test-code/src/main/java/com/launchdarkly/sdk/android/MockPlatformState.java b/shared-test-code/src/main/java/com/launchdarkly/sdk/android/MockPlatformState.java index 1b08ef95d..8d9bccdf3 100644 --- a/shared-test-code/src/main/java/com/launchdarkly/sdk/android/MockPlatformState.java +++ b/shared-test-code/src/main/java/com/launchdarkly/sdk/android/MockPlatformState.java @@ -16,6 +16,8 @@ public class MockPlatformState implements PlatformState { private volatile boolean foreground = true; private volatile boolean networkAvailable = true; + private volatile File noBackupFilesDir = null; + private volatile String processName = "test"; @Override public boolean isNetworkAvailable() { @@ -105,6 +107,28 @@ public File getCacheDir() { return new File(System.getProperty("java.io.tmpdir")); } + @Override + public File getNoBackupFilesDir() { + return noBackupFilesDir != null ? noBackupFilesDir : getCacheDir(); + } + + public void setNoBackupFilesDir(File noBackupFilesDir) { + this.noBackupFilesDir = noBackupFilesDir; + } + + @Override + public String getProcessName() { + return processName; + } + + /** + * Lets a test act as a different process of a multi-process application while sharing the same + * directory, which is the situation the on-disk event log has to be safe in. + */ + public void setProcessName(String processName) { + this.processName = processName; + } + @Override public void close() throws IOException {} } diff --git a/test-app/README.md b/test-app/README.md new file mode 100644 index 000000000..e39f8d42f --- /dev/null +++ b/test-app/README.md @@ -0,0 +1,37 @@ +# LaunchDarkly Android SDK test app + +This app is an internal quality harness. For customer integration examples, use `example`. + +Add the following to the repository's root `local.properties`: + +```properties +launchdarkly.mobileKey=mob-... +launchdarkly.environment=production +``` + +Set `launchdarkly.environment=staging` to use LaunchDarkly's staging endpoints. + +## Event-loss scenarios + +Create a boolean flag named `kill-flag`, or enter another flag key in the app. Tap +**Eval+track+kill** to evaluate the flag, track a stand-in error event, request a flush, and +terminate the process five seconds later. This exercises the interval between recording and +delivery without Android lifecycle callbacks masking the result. + +The two immediate controls compare exits that application code can and cannot observe: + +- **Eval+Kill now** records the same pair and sends `SIGKILL` immediately. No handler or SDK code + can run before the process ends. +- **Eval+Crash now** throws an uncaught exception immediately after recording. The installed crash + handler calls `flushAndWait` with a two-second budget before delegating to Android's handler. + +Tier 3 configures `EventPersistence.IMMEDIATE`. With that setting, events recorded by +**Eval+Kill now** should be recovered and delivered after the next launch even though no process +code ran on exit. + +## Over-refresh scenario + +Create a boolean flag named `trackevents-test` with event tracking enabled. Tap +**Over-Refresh Eval** to compare the average evaluation cost while filling the event capacity and +after it is full. The loop intentionally runs on the main thread and is an internal stress test, +not an application integration pattern. diff --git a/test-app/build.gradle b/test-app/build.gradle new file mode 100644 index 000000000..d5b3d28a9 --- /dev/null +++ b/test-app/build.gradle @@ -0,0 +1,50 @@ +import java.util.Properties + +plugins { + id("com.android.application") + // Method counts are an internal quality signal rather than customer setup. + id("com.getkeepsafe.dexcount") +} + +// local.properties is not checked in, so it is where machine-specific settings such as the mobile +// key belong. See test-app/README.md for the keys this app reads. +def localProperties = new Properties() +def localPropertiesFile = rootProject.file("local.properties") +if (localPropertiesFile.exists()) { + localPropertiesFile.withInputStream { localProperties.load(it) } +} + +android { + namespace "com.launchdarkly.sdk.testapp" + compileSdk = 34 + + defaultConfig { + applicationId = "com.launchdarkly.sdk.testapp" + minSdk = 21 + targetSdk = 34 + versionCode = 1 + versionName = "1.0" + + buildConfigField("String", "MOBILE_KEY", + "\"${localProperties.getProperty('launchdarkly.mobileKey', '')}\"") + buildConfigField("String", "LD_ENVIRONMENT", + "\"${localProperties.getProperty('launchdarkly.environment', 'production')}\"") + } + + buildFeatures { + buildConfig = true + } + + buildTypes { + release { + minifyEnabled = true + proguardFiles(getDefaultProguardFile("proguard-android.txt"), "proguard-rules.pro") + } + } +} + +dependencies { + implementation("androidx.appcompat:appcompat:1.2.0") + implementation("com.jakewharton.timber:timber:5.0.1") + implementation(project(":launchdarkly-android-client-sdk")) +} diff --git a/test-app/proguard-rules.pro b/test-app/proguard-rules.pro new file mode 100644 index 000000000..8788708d0 --- /dev/null +++ b/test-app/proguard-rules.pro @@ -0,0 +1,2 @@ +# Allow compile-time-only error-prone annotations to be stripped without breaking shrink. +-dontwarn com.google.errorprone.annotations.** diff --git a/test-app/src/main/AndroidManifest.xml b/test-app/src/main/AndroidManifest.xml new file mode 100644 index 000000000..b9729d0ff --- /dev/null +++ b/test-app/src/main/AndroidManifest.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + diff --git a/test-app/src/main/java/com/launchdarkly/sdk/testapp/ExposureCountingHook.java b/test-app/src/main/java/com/launchdarkly/sdk/testapp/ExposureCountingHook.java new file mode 100644 index 000000000..4ace6bce6 --- /dev/null +++ b/test-app/src/main/java/com/launchdarkly/sdk/testapp/ExposureCountingHook.java @@ -0,0 +1,64 @@ +package com.launchdarkly.sdk.testapp; + +import com.launchdarkly.sdk.EvaluationDetail; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.android.integrations.EvaluationSeriesContext; +import com.launchdarkly.sdk.android.integrations.Hook; + +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Counts the evaluation series stages it observes, so the example can show what exposure + * deduplication does. Deduplication is configured at registration by wrapping this hook in a + * {@link com.launchdarkly.sdk.android.integrations.DedupingHook}, the same way a customer would + * wrap any other hook. + *

+ * Deduplication skips the whole series, so both counts stay equal and both stop climbing while + * repeated evaluations resolve to the same result. + */ +class ExposureCountingHook extends Hook { + private final String label; + private final Runnable onStage; + private final AtomicInteger befores = new AtomicInteger(); + private final AtomicInteger afters = new AtomicInteger(); + + /** + * @param label a name for this hook, shown in the app's dedupe status + * @param onStage run after each stage so the app can refresh its display + */ + ExposureCountingHook(String label, Runnable onStage) { + super(label); + this.label = label; + this.onStage = onStage; + } + + @Override + public Map beforeEvaluation(EvaluationSeriesContext seriesContext, Map seriesData) { + befores.incrementAndGet(); + onStage.run(); + return seriesData; + } + + @Override + public Map afterEvaluation(EvaluationSeriesContext seriesContext, Map seriesData, + EvaluationDetail evaluationDetail) { + afters.incrementAndGet(); + onStage.run(); + return seriesData; + } + + /** + * @param windowMillis the dedupe window this hook was registered with, for the status line + * @return a line describing this hook's window and how many evaluations have reached it + */ + String status(int windowMillis) { + return String.format(Locale.US, "%s (%d ms): %d (before %d / after %d)", + label, + windowMillis, + afters.get(), + befores.get(), + afters.get()); + } +} diff --git a/test-app/src/main/java/com/launchdarkly/sdk/testapp/MainActivity.java b/test-app/src/main/java/com/launchdarkly/sdk/testapp/MainActivity.java new file mode 100644 index 000000000..6861307cf --- /dev/null +++ b/test-app/src/main/java/com/launchdarkly/sdk/testapp/MainActivity.java @@ -0,0 +1,552 @@ +package com.launchdarkly.sdk.testapp; + +import android.content.SharedPreferences; +import android.os.Bundle; +import android.os.Handler; +import android.os.Looper; +import android.view.View; +import android.widget.AdapterView; +import android.widget.ArrayAdapter; +import android.widget.Button; +import android.widget.EditText; +import android.widget.Spinner; +import android.widget.Switch; +import android.widget.TextView; +import android.widget.Toast; + +import androidx.appcompat.app.AppCompatActivity; + +import com.launchdarkly.sdk.LDContext; +import com.launchdarkly.sdk.android.Components; +import com.launchdarkly.sdk.android.ConnectionInformation; +import com.launchdarkly.sdk.android.LDAllFlagsListener; +import com.launchdarkly.sdk.android.LDClient; +import com.launchdarkly.sdk.android.LDConfig; +import com.launchdarkly.sdk.android.LDConfig.Builder.AutoEnvAttributes; +import com.launchdarkly.sdk.android.LDFailure; +import com.launchdarkly.sdk.android.LDStatusListener; +import com.launchdarkly.sdk.android.integrations.DedupingHook; +import com.launchdarkly.sdk.android.integrations.EventPersistence; +import com.launchdarkly.sdk.android.integrations.EventProcessorBuilder; +import com.launchdarkly.sdk.android.integrations.LDCrashHandler; + +import java.io.IOException; +import java.util.Date; +import java.util.Locale; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import timber.log.Timber; + +public class MainActivity extends AppCompatActivity { + + // Two hooks with different windows, to show that each one is deduplicated on its own. + private static final int FAST_DEDUPE_WINDOW_MILLIS = 5_000; + private static final int SLOW_DEDUPE_WINDOW_MILLIS = 10_000; + + // The staging hosts mirror the production ones in StandardEndpoints under the ld-stg domain. + private static final String STAGING_DOMAIN = "ld-stg.launchdarkly.com"; + + private static final String DEFAULT_USER_KEY = "user key"; + + /** How long startup blocks waiting for the first flags to arrive. */ + private static final int INIT_WAIT_SECONDS = 10; + + private static final String PREFERENCES_NAME = "LaunchDarkly-testapp"; + private static final String PERSISTENCE_PREFERENCE = "eventPersistence"; + private static final EventPersistence DEFAULT_PERSISTENCE = EventPersistence.IMMEDIATE; + + /** + * The default of {@code LDConfig.Builder#events} capacity. Past this the event store is full until a + * flush empties it, so every further full event is dropped. + */ + private static final int EVENT_CAPACITY = EventProcessorBuilder.DEFAULT_CAPACITY; + + /** + * Enough evaluations past capacity for the average to settle, and few enough that even a slow device + * finishes the tap well inside the ANR window. + */ + private static final int OVER_CAPACITY_EVALUATIONS = 2000; + + /** Null while a persistence change restarts the client, which makes every button a no-op meanwhile. */ + private volatile LDClient ldClient; + private EventPersistence eventPersistence; + private LDStatusListener ldStatusListener; + private LDAllFlagsListener allFlagsListener; + + private final ExposureCountingHook fastHook = + new ExposureCountingHook("fast", this::updateDedupeStatus); + private final ExposureCountingHook slowHook = + new ExposureCountingHook("slow", this::updateDedupeStatus); + private final AtomicInteger evaluationsRequested = new AtomicInteger(); + + private static boolean isStaging() { + return "staging".equalsIgnoreCase(BuildConfig.LD_ENVIRONMENT); + } + + private void updateDedupeStatus() { + if (Looper.myLooper() != MainActivity.this.getMainLooper()) { + new Handler(MainActivity.this.getMainLooper()).post(this::updateDedupeStatus); + return; + } + + String result = String.format(Locale.US, + "Environment: %s\nEvaluations requested: %d\n%s\n%s", + isStaging() ? "staging" : "production", + evaluationsRequested.get(), + fastHook.status(FAST_DEDUPE_WINDOW_MILLIS), + slowHook.status(SLOW_DEDUPE_WINDOW_MILLIS)); + ((TextView) MainActivity.this.findViewById(R.id.dedupe_status)).setText(result); + } + + private void updateStatusString(final ConnectionInformation connectionInformation) { + if (Looper.myLooper() != MainActivity.this.getMainLooper()) { + new Handler(MainActivity.this.getMainLooper()).post(() -> updateStatusString(connectionInformation)); + } else { + TextView connection = MainActivity.this.findViewById(R.id.connection_status); + Long lastSuccess = connectionInformation.getLastSuccessfulConnection(); + Long lastFailure = connectionInformation.getLastFailedConnection(); + + String result = String.format(Locale.US, "Mode: %s\nSuccess at: %s\nFailure at: %s\nFailure type: %s", + connectionInformation.getConnectionMode().toString(), + lastSuccess == null ? "Never" : new Date(lastSuccess).toString(), + lastFailure == null ? "Never" : new Date(lastFailure).toString(), + connectionInformation.getLastFailure() != null ? + connectionInformation.getLastFailure().getFailureType() + : ""); + connection.setText(result); + } + } + + @Override + public void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + + setupEval(); + setupFlushButton(); + setupTrackButton(); + setupIdentifyButton(); + setupKillUnsentButton(); + setupKillNowButton(); + setupCrashNowButton(); + setupOverRefreshButton(); + setupOfflineSwitch(); + setupPersistenceSpinner(); + // Rescues the events for "Eval+Crash now" and cannot run for "Eval+Kill now", which is what + // makes the pair worth pressing. + LDCrashHandler.install(2, TimeUnit.SECONDS); + setupListeners(); + updateDedupeStatus(); + + if (BuildConfig.MOBILE_KEY.isEmpty()) { + String message = "Set launchdarkly.mobileKey in local.properties and rebuild."; + Timber.e(message); + Toast.makeText(this, message, Toast.LENGTH_LONG).show(); + return; + } + + ldClient = startClient(eventPersistence); + } + + /** + * Blocks for up to {@link #INIT_WAIT_SECONDS} waiting for flags, and returns the client either way: if + * they have not arrived by then, it is usable with whatever it has cached. + */ + private LDClient startClient(EventPersistence persistence) { + LDConfig.Builder configBuilder = new LDConfig.Builder(AutoEnvAttributes.Enabled) + .mobileKey(BuildConfig.MOBILE_KEY) + .http( + Components.httpConfiguration().useReport(false) + // change useReport to `true` if the request is to be REPORT'ed instead of GET'ed + ) + .events( + Components.sendEvents().eventPersistence(persistence) + ) + .hooks( + // Same shape a customer uses for any hook: wrap it at registration. Each + // wrapper has its own window, so neither suppresses the other. + Components.hooks() + .addHook(new DedupingHook(fastHook, FAST_DEDUPE_WINDOW_MILLIS)) + .addHook(new DedupingHook(slowHook, SLOW_DEDUPE_WINDOW_MILLIS)) + ); + + if (isStaging()) { + configBuilder.serviceEndpoints( + Components.serviceEndpoints() + .streaming("https://clientstream." + STAGING_DOMAIN) + .polling("https://clientsdk." + STAGING_DOMAIN) + .events("https://mobile." + STAGING_DOMAIN) + ); + } + + LDConfig ldConfig = configBuilder.build(); + + LDContext context = LDContext.builder(DEFAULT_USER_KEY) + .set("email", "fake@example.com") + .build(); + + LDClient client = LDClient.init(this.getApplication(), ldConfig, context, INIT_WAIT_SECONDS); + Timber.i("client started with event persistence %s", persistence); + updateStatusString(client.getConnectionInformation()); + client.registerStatusListener(ldStatusListener); + client.registerAllFlagsListener(allFlagsListener); + return client; + } + + /** + * The choice is saved rather than held in memory, because the kill and crash buttons are only + * answered on the next launch, and that launch has to read the disk the same way: under + * {@link EventPersistence#DISABLED} the events the dead process left there are not delivered. + */ + private void setupPersistenceSpinner() { + final SharedPreferences preferences = getSharedPreferences(PREFERENCES_NAME, MODE_PRIVATE); + eventPersistence = parsePersistence(preferences.getString(PERSISTENCE_PREFERENCE, null)); + + final Spinner spinner = findViewById(R.id.persistence_spinner); + ArrayAdapter adapter = new ArrayAdapter<>(this, + android.R.layout.simple_spinner_item, EventPersistence.values()); + adapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); + spinner.setAdapter(adapter); + spinner.setSelection(eventPersistence.ordinal(), false); + + spinner.setOnItemSelectedListener(new AdapterView.OnItemSelectedListener() { + @Override + public void onItemSelected(AdapterView parent, View view, int position, long id) { + EventPersistence chosen = (EventPersistence) parent.getItemAtPosition(position); + if (chosen == eventPersistence) { + return; + } + eventPersistence = chosen; + preferences.edit().putString(PERSISTENCE_PREFERENCE, chosen.name()).apply(); + restartClient(spinner, chosen); + } + + @Override + public void onNothingSelected(AdapterView parent) { + } + }); + } + + private static EventPersistence parsePersistence(String name) { + if (name != null) { + try { + return EventPersistence.valueOf(name); + } catch (IllegalArgumentException e) { + Timber.w("ignoring saved event persistence %s", name); + } + } + return DEFAULT_PERSISTENCE; + } + + /** + * Off the main thread, because closing waits on the event store and starting waits for flags. The + * spinner stays disabled until it is done, so restarts never overlap. + */ + private void restartClient(final Spinner spinner, final EventPersistence persistence) { + if (BuildConfig.MOBILE_KEY.isEmpty()) { + return; + } + final LDClient previous = ldClient; + ldClient = null; + spinner.setEnabled(false); + new Thread(() -> { + if (previous != null) { + previous.unregisterStatusListener(ldStatusListener); + previous.unregisterAllFlagsListener(allFlagsListener); + try { + previous.close(); + } catch (IOException e) { + Timber.w(e, "closing the client for a persistence change"); + } + } + LDClient restarted = startClient(persistence); + new Handler(getMainLooper()).post(() -> { + ldClient = restarted; + spinner.setEnabled(true); + if (((Switch) findViewById(R.id.offlineSwitch)).isChecked()) { + restarted.setOffline(); + } + Toast.makeText(this, "Restarted with " + persistence, Toast.LENGTH_SHORT).show(); + }); + }, "testapp-restart-client").start(); + } + + private void setupListeners() { + ldStatusListener = new LDStatusListener() { + @Override + public void onConnectionModeChanged(final ConnectionInformation connectionInformation) { + updateStatusString(connectionInformation); + } + + @Override + public void onInternalFailure(final LDFailure ldFailure) { + new Handler(MainActivity.this.getMainLooper()).post(() -> { + Toast.makeText(MainActivity.this, ldFailure.toString(), Toast.LENGTH_SHORT).show(); + }); + MainActivity.this.doSafeClientAction(() -> updateStatusString(ldClient.getConnectionInformation())); + } + }; + + allFlagsListener = flagKey -> { + new Handler(MainActivity.this.getMainLooper()).post(() -> { + StringBuilder flags = new StringBuilder("Updated flags: "); + for (String flag : flagKey) { + flags.append(flag).append(" "); + } + Toast.makeText(MainActivity.this, flags.toString(), Toast.LENGTH_SHORT).show(); + }); + MainActivity.this.doSafeClientAction(() -> updateStatusString(ldClient.getConnectionInformation())); + }; + } + + private void setupFlushButton() { + Button flushButton = findViewById(R.id.flush_button); + flushButton.setOnClickListener(v -> { + Timber.i("flush onClick"); + MainActivity.this.doSafeClientAction(() -> ldClient.flush()); + }); + } + + private interface LDClientAction { + void call(); + } + + private void doSafeClientAction(LDClientAction function) { + if (ldClient != null) { + function.call(); + } + } + + private interface LDClientGet { + V get(); + } + + private V doSafeClientGet(LDClientGet function) { + return ldClient != null ? function.get() : null; + } + + private void setupTrackButton() { + Button trackButton = findViewById(R.id.track_button); + trackButton.setOnClickListener(v -> { + Timber.i("track onClick"); + MainActivity.this.doSafeClientAction(() -> ldClient.track("Android event name")); + }); + } + + /** + * The flag the kill and crash buttons evaluate: whatever is typed in the feature key field, or a + * default, so the buttons work without anything being typed first. + */ + private String flagKeyToKillOver() { + String typedKey = ((EditText) findViewById(R.id.feature_flag_key)).getText().toString().trim(); + return typedKey.isEmpty() ? "kill-flag" : typedKey; + } + + /** + * Records the pair whose survival is in question: an evaluation, which is the exposure, and a + * track, standing in for the error an application reports just before it dies. + * + *

Returns false when there is no client, in which case nothing was recorded and ending the + * process would demonstrate nothing. + */ + private boolean recordExposureAndError(String flagKey) { + if (ldClient == null) { + return false; + } + ldClient.boolVariation(flagKey, false); + ldClient.track("$ld:telemetry:error"); + return true; + } + + /** + * Reproduces in-memory event loss: evaluate (exposure) and track (stand-in for an error), + * wait 5s so both calls are queued, then kill the process before the 30s flush. + * {@code finish()} or backgrounding would run the SDK's background flush, so this uses + * {@link android.os.Process#killProcess}. + */ + private void setupKillUnsentButton() { + Button killUnsentButton = findViewById(R.id.kill_unsent_button); + killUnsentButton.setOnClickListener(v -> { + final String flagKey = flagKeyToKillOver(); + Timber.w("eval+track+kill flag=%s", flagKey); + if (!recordExposureAndError(flagKey)) { + return; + } + ldClient.flush(); + new Handler(Looper.getMainLooper()).postDelayed( + () -> android.os.Process.killProcess(android.os.Process.myPid()), + 5_000); + }); + } + + /** + * The same sequence with nothing at all between the track and the process dying: no flush to + * deliver the events, no delay for a timer to fire in, and SIGKILL to itself, which cannot be + * caught, so no part of the SDK gets to run on the way out. + * + *

Whether the exposure and the track are reported therefore says exactly one thing: whether + * recording them had already put them somewhere that outlives the process. They should arrive on + * the next launch of the app, not this one. + */ + private void setupKillNowButton() { + Button killNowButton = findViewById(R.id.kill_now_button); + killNowButton.setOnClickListener(v -> { + final String flagKey = flagKeyToKillOver(); + Timber.w("eval+track+kill now flag=%s", flagKey); + if (!recordExposureAndError(flagKey)) { + return; + } + android.os.Process.killProcess(android.os.Process.myPid()); + }); + } + + /** + * The same again, ending in an uncaught exception instead of a signal the process never sees. + * + *

This is the shape a customer report takes: app code fails immediately after reporting the + * failure. Unlike SIGKILL, an uncaught exception runs the default handler before the process + * goes, so this is the one variant an application can rescue on its own, which + * {@link LDCrashHandler} does by calling {@link LDClient#flushAndWait} from there. So these + * events should arrive and the ones from the button next to it should not. + */ + private void setupCrashNowButton() { + Button crashNowButton = findViewById(R.id.crash_now_button); + crashNowButton.setOnClickListener(v -> { + final String flagKey = flagKeyToKillOver(); + Timber.w("eval+track+crash now flag=%s", flagKey); + if (!recordExposureAndError(flagKey)) { + return; + } + throw new RuntimeException( + "Eval+Crash: deliberate uncaught exception immediately after track, to test event persistence"); + }); + } + + /** + * Reproduces the over-refresh pathology: a UI that re-evaluates one flag far more often than anything + * about it changed -- a recomposition or layout loop -- against a flag whose events LaunchDarkly is + * tracking. Runs on the main thread, because that is where the loop it stands in for runs. + * + *

The comparison between the two phases is the point rather than either number on its own. The first + * {@link #EVENT_CAPACITY} evaluations have room in the event store; every one after that is dropped, + * because nothing empties it until a flush. So the two phases costing the same per evaluation means the + * SDK is serializing events it then discards, which is the whole of the waste. A cost of a couple of + * microseconds instead means the flag has event tracking off, and this measured the summary counters + * rather than the thing in question. + */ + private void setupOverRefreshButton() { + Button overRefreshButton = findViewById(R.id.over_refresh_button); + overRefreshButton.setOnClickListener(v -> { + if (ldClient == null) { + return; + } + final String flagKey = "trackevents-test"; + Timber.w("over-refresh eval flag=%s", flagKey); + + double filling = averageMicroseconds(flagKey, EVENT_CAPACITY); + double overCapacity = averageMicroseconds(flagKey, OVER_CAPACITY_EVALUATIONS); + + String result = String.format(Locale.US, + "Over-refresh %s\n%d filling: %.0f \u00b5s/eval\n%d over capacity: %.0f \u00b5s/eval", + flagKey, + EVENT_CAPACITY, filling, + OVER_CAPACITY_EVALUATIONS, overCapacity); + Timber.w(result); + ((TextView) findViewById(R.id.result_textView)).setText(result); + }); + } + + /** + * @return the mean wall time, in microseconds, of {@code count} evaluations of the flag + */ + private double averageMicroseconds(String flagKey, int count) { + long start = System.nanoTime(); + for (int i = 0; i < count; i++) { + ldClient.boolVariation(flagKey, false); + } + return (System.nanoTime() - start) / 1_000.0 / count; + } + + private void setupIdentifyButton() { + Button identify = findViewById(R.id.identify_button); + identify.setOnClickListener(v -> { + Timber.i("identify onClick"); + String typedKey = ((EditText) MainActivity.this.findViewById(R.id.userKey_editText)) + .getText().toString().trim(); + // An empty key builds an invalid context, which identify rejects before it resets the + // hooks' dedupe caches. Fall back to the key the client started with, so identifying to + // an unchanged context still demonstrates that the reset happens either way. + final String userKey = typedKey.isEmpty() ? DEFAULT_USER_KEY : typedKey; + final LDContext updatedContext = LDContext.create(userKey); + MainActivity.this.doSafeClientAction(() -> { + ldClient.identify(updatedContext); + }); + MainActivity.this.updateDedupeStatus(); + }); + } + + private void setupOfflineSwitch() { + Switch offlineSwitch = findViewById(R.id.offlineSwitch); + offlineSwitch.setOnCheckedChangeListener((compoundButton, isChecked) -> + MainActivity.this.doSafeClientAction(isChecked ? () -> ldClient.setOffline() : () -> ldClient.setOnline()) + ); + } + + private void setupEval() { + final Spinner spinner = findViewById(R.id.type_spinner); + ArrayAdapter adapter = ArrayAdapter.createFromResource(this, + R.array.types_array, android.R.layout.simple_spinner_item); + adapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); + spinner.setAdapter(adapter); + + Button evalButton = findViewById(R.id.eval_button); + evalButton.setOnClickListener(v -> { + Timber.i("eval onClick"); + final String flagKey = ((EditText) MainActivity.this.findViewById(R.id.feature_flag_key)).getText().toString(); + evaluationsRequested.incrementAndGet(); + + String type = spinner.getSelectedItem().toString(); + final String result; + String logResult; + switch (type) { + case "String": + result = MainActivity.this.doSafeClientGet(() -> ldClient.stringVariation(flagKey, "default")); + logResult = result == null ? "no result" : result; + Timber.i(logResult); + ((TextView) MainActivity.this.findViewById(R.id.result_textView)).setText(result); + MainActivity.this.doSafeClientAction(() -> { + ldClient.registerFeatureFlagListener(flagKey, flagKey1 -> { + evaluationsRequested.incrementAndGet(); + ((TextView) MainActivity.this.findViewById(R.id.result_textView)) + .setText(ldClient.stringVariation(flagKey1, "default")); + MainActivity.this.updateDedupeStatus(); + }); + }); + MainActivity.this.updateDedupeStatus(); + return; + case "Boolean": + result = MainActivity.this.doSafeClientGet(() -> String.valueOf(ldClient.boolVariation(flagKey, false))); + break; + case "Integer": + result = MainActivity.this.doSafeClientGet(() -> String.valueOf(ldClient.intVariation(flagKey, 0))); + break; + case "Float": + result = MainActivity.this.doSafeClientGet(() -> String.valueOf(ldClient.doubleVariation(flagKey, 0.0))); + break; + case "Value": + result = MainActivity.this.doSafeClientGet(() -> String.valueOf(ldClient.jsonValueVariation(flagKey, null))); + break; + default: + result = null; + break; + } + + logResult = result == null ? "no result" : result; + Timber.i(logResult); + ((TextView) MainActivity.this.findViewById(R.id.result_textView)).setText(result); + MainActivity.this.updateDedupeStatus(); + }); + } + +} diff --git a/test-app/src/main/res/layout/activity_main.xml b/test-app/src/main/res/layout/activity_main.xml new file mode 100644 index 000000000..ed7147dc3 --- /dev/null +++ b/test-app/src/main/res/layout/activity_main.xml @@ -0,0 +1,252 @@ + + + +