diff --git a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/TestHelper.kt b/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/TestHelper.kt index 7f0dfc8c955..8ef74cae3c9 100644 --- a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/TestHelper.kt +++ b/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/TestHelper.kt @@ -4,9 +4,9 @@ import com.apollographql.apollo.api.ApolloResponse import com.apollographql.apollo.api.Operation import io.sentry.JsonSerializer import io.sentry.ProfileChunk +import io.sentry.SentryEnvelope import io.sentry.SentryEnvelopeHeader import io.sentry.SentryEvent -import io.sentry.SentryItemType import io.sentry.SentryLogEvents import io.sentry.SentryMetricsEvents import io.sentry.SentryOptions @@ -14,10 +14,22 @@ import io.sentry.protocol.FeatureFlag import io.sentry.protocol.SentrySpan import io.sentry.protocol.SentryTransaction import io.sentry.systemtest.graphql.GraphqlTestClient -import java.io.BufferedReader -import java.io.ByteArrayInputStream +import io.sentry.test.event +import io.sentry.test.hasExceptionMessage +import io.sentry.test.hasFlag +import io.sentry.test.hasLogWithAttribute +import io.sentry.test.hasLogWithBody +import io.sentry.test.hasMetric +import io.sentry.test.hasMetricWithAttribute +import io.sentry.test.hasOp +import io.sentry.test.hasSpanWithOp +import io.sentry.test.hasSpanWithOpAndDescription +import io.sentry.test.hasTraceId +import io.sentry.test.logs +import io.sentry.test.metrics +import io.sentry.test.profileChunk +import io.sentry.test.transaction import java.io.File -import java.io.InputStreamReader import java.io.PrintWriter import kotlin.test.assertEquals import kotlin.test.assertFalse @@ -107,24 +119,9 @@ class TestHelper(backendUrl: String) { envelopeString: String, callback: ((SentryLogEvents, SentryEnvelopeHeader) -> Boolean), ): Boolean { - val deserializeEnvelope = jsonSerializer.deserializeEnvelope(envelopeString.byteInputStream()) - if (deserializeEnvelope == null) { - return false - } - - val envelopeHeader = deserializeEnvelope.header - - val logsItem = deserializeEnvelope.items.firstOrNull { it.header.type == SentryItemType.Log } - if (logsItem == null) { - return false - } - - val logs = logsItem.getLogs(jsonSerializer) - if (logs == null) { - return false - } - - return callback(logs, envelopeHeader) + val envelope = deserialize(envelopeString) ?: return false + val logs = envelope.logs() ?: return false + return callback(logs, envelope.header) } fun doesContainMetric( @@ -133,22 +130,7 @@ class TestHelper(backendUrl: String) { type: String, value: Double, unit: String? = null, - ): Boolean { - val metricItem = - metrics.items.firstOrNull { metricItem -> - metricItem.name == name && - metricItem.type == type && - metricItem.value == value && - (unit == null || metricItem.unit == unit) - } - if (metricItem == null) { - println("Unable to find metric item with name $name, type $type and value $value in metrics:") - logObject(metrics) - return false - } - - return true - } + ): Boolean = metrics.hasMetric(name, type, value, unit) fun ensureMetricsReceived(callback: ((SentryMetricsEvents, SentryEnvelopeHeader) -> Boolean)) { ensureEnvelopeReceived { envelopeString -> checkIfMetricsMatch(envelopeString, callback) } @@ -158,172 +140,49 @@ class TestHelper(backendUrl: String) { envelopeString: String, callback: ((SentryMetricsEvents, SentryEnvelopeHeader) -> Boolean), ): Boolean { - val deserializeEnvelope = jsonSerializer.deserializeEnvelope(envelopeString.byteInputStream()) - if (deserializeEnvelope == null) { - return false - } - - val envelopeHeader = deserializeEnvelope.header - - val metricsItem = - deserializeEnvelope.items.firstOrNull { it.header.type == SentryItemType.TraceMetric } - if (metricsItem == null) { - return false - } - - val metrics = metricsItem.getMetrics(jsonSerializer) - if (metrics == null) { - return false - } - - return callback(metrics, envelopeHeader) + val envelope = deserialize(envelopeString) ?: return false + val metrics = envelope.metrics() ?: return false + return callback(metrics, envelope.header) } - fun doesContainLogWithBody(logs: SentryLogEvents, body: String): Boolean { - val logItem = logs.items.firstOrNull { logItem -> logItem.body == body } - if (logItem == null) { - println("Unable to find log item with body $body in logs:") - logObject(logs) - return false - } - - return true - } + fun doesContainLogWithBody(logs: SentryLogEvents, body: String): Boolean = + logs.hasLogWithBody(body) fun doesLogWithBodyHaveAttribute( logs: SentryLogEvents, body: String, attributeKey: String, attributeValue: Any?, - ): Boolean { - val logItem = logs.items.firstOrNull { logItem -> logItem.body == body } - if (logItem == null) { - println("Unable to find log item with body $body in logs:") - logObject(logs) - return false - } - - val attr = logItem.attributes?.get(attributeKey) - if (attr == null) { - println("Unable to find attribute $attributeKey on log with body $body:") - logObject(logItem) - return false - } - - if (attr.value != attributeValue) { - println( - "Attribute $attributeKey has value ${attr.value} but expected $attributeValue on log with body $body:" - ) - logObject(logItem) - return false - } - - return true - } + ): Boolean = logs.hasLogWithAttribute(body, attributeKey, attributeValue) fun doesMetricHaveAttribute( metrics: SentryMetricsEvents, metricName: String, attributeKey: String, attributeValue: Any?, - ): Boolean { - val metricItem = metrics.items.firstOrNull { it.name == metricName } - if (metricItem == null) { - println("Unable to find metric with name $metricName in metrics:") - logObject(metrics) - return false - } - - val attr = metricItem.attributes?.get(attributeKey) - if (attr == null) { - println("Unable to find attribute $attributeKey on metric $metricName:") - logObject(metricItem) - return false - } - - if (attr.value != attributeValue) { - println( - "Attribute $attributeKey has value ${attr.value} but expected $attributeValue on metric $metricName:" - ) - logObject(metricItem) - return false - } - - return true - } + ): Boolean = metrics.hasMetricWithAttribute(metricName, attributeKey, attributeValue) private fun checkIfTransactionMatches( envelopeString: String, callback: ((SentryTransaction, SentryEnvelopeHeader) -> Boolean), ): Boolean { - val deserializeEnvelope = jsonSerializer.deserializeEnvelope(envelopeString.byteInputStream()) - if (deserializeEnvelope == null) { - return false - } - - val envelopeHeader = deserializeEnvelope.header - - val transactionItem = - deserializeEnvelope.items.firstOrNull { it.header.type == SentryItemType.Transaction } - if (transactionItem == null) { - return false - } - - val transaction = transactionItem.getTransaction(jsonSerializer) - if (transaction == null) { - return false - } - - return callback(transaction, envelopeHeader) + val envelope = deserialize(envelopeString) ?: return false + val transaction = envelope.transaction() ?: return false + return callback(transaction, envelope.header) } private fun checkIfProfileMatches( envelopeString: String, callback: ((ProfileChunk, SentryEnvelopeHeader) -> Boolean), ): Boolean { - val deserializeEnvelope = jsonSerializer.deserializeEnvelope(envelopeString.byteInputStream()) - if (deserializeEnvelope == null) { - return false - } - - val envelopeHeader = deserializeEnvelope.header - - val profileChunkItem = - deserializeEnvelope.items.firstOrNull { it.header.type == SentryItemType.ProfileChunk } - - if (profileChunkItem == null) { - return false - } - - val chunk = - BufferedReader(InputStreamReader(ByteArrayInputStream(profileChunkItem.data), Charsets.UTF_8)) - .use { eventReader -> jsonSerializer.deserialize(eventReader, ProfileChunk::class.java) } - - if (chunk == null) { - return false - } - - return callback(chunk, envelopeHeader) + val envelope = deserialize(envelopeString) ?: return false + val chunk = envelope.profileChunk() ?: return false + return callback(chunk, envelope.header) } fun ensureErrorReceived(callback: ((SentryEvent) -> Boolean)) { ensureEnvelopeReceived(retryCount = 3) { envelopeString -> - val deserializeEnvelope = jsonSerializer.deserializeEnvelope(envelopeString.byteInputStream()) - if (deserializeEnvelope == null) { - return@ensureEnvelopeReceived false - } - - val errorItem = - deserializeEnvelope.items.firstOrNull { it.header.type == SentryItemType.Event } - if (errorItem == null) { - return@ensureEnvelopeReceived false - } - - val error = errorItem.getEvent(jsonSerializer) - if (error == null) { - return@ensureEnvelopeReceived false - } - + val error = deserialize(envelopeString)?.event() ?: return@ensureEnvelopeReceived false val callbackResult = callback(error) if (!callbackResult) { println("found an error event but it did not match:") @@ -351,10 +210,17 @@ class TestHelper(backendUrl: String) { fun logObject(obj: Any?) { obj ?: return - PrintWriter(System.out).use { jsonSerializer.serialize(obj, it) } + // Flush rather than close: closing the writer would close System.out. + PrintWriter(System.out).apply { + jsonSerializer.serialize(obj, this) + flush() + } println() } + private fun deserialize(envelopeString: String): SentryEnvelope? = + jsonSerializer.deserializeEnvelope(envelopeString.byteInputStream()) + fun ensureNoErrors(response: ApolloResponse?) { response ?: throw RuntimeException("no response") assertFalse(response.hasErrors()) @@ -365,156 +231,39 @@ class TestHelper(backendUrl: String) { assertEquals(errorCount, response.errors?.size) } - fun doesTransactionContainSpanWithOp(transaction: SentryTransaction, op: String): Boolean { - val span = transaction.spans.firstOrNull { span -> span.op == op } - if (span == null) { - println("Unable to find span with op $op in transaction:") - logObject(transaction) - return false - } - - return true - } + fun doesTransactionContainSpanWithOp(transaction: SentryTransaction, op: String): Boolean = + transaction.hasSpanWithOp(op) fun doesTransactionContainSpanWithOpAndDescription( transaction: SentryTransaction, op: String, description: String, - ): Boolean { - val span = - transaction.spans.firstOrNull { span -> span.op == op && span.description == description } - if (span == null) { - println("Unable to find span with op $op and description $description in transaction:") - logObject(transaction) - return false - } - - return true - } - - fun doesTransactionContainSpanWithDescription( - transaction: SentryTransaction, - description: String, - ): Boolean { - val span = transaction.spans.firstOrNull { span -> span.description == description } - if (span == null) { - println("Unable to find span with description $description in transaction:") - logObject(transaction) - return false - } - - return true - } - - fun doesTransactionHaveTraceId(transaction: SentryTransaction, traceId: String): Boolean { - val spanContext = transaction.contexts.trace - if (spanContext?.traceId?.toString() != traceId) { - println("Unable to find trace ID $traceId in transaction:") - logObject(transaction) - return false - } + ): Boolean = transaction.hasSpanWithOpAndDescription(op, description) - return true - } - - fun doesTransactionHaveOp(transaction: SentryTransaction, op: String): Boolean { - val matches = transaction.contexts.trace?.operation == op - if (!matches) { - println("Unable to find transaction with op $op:") - logObject(transaction) - return false - } + fun doesTransactionHaveTraceId(transaction: SentryTransaction, traceId: String): Boolean = + transaction.hasTraceId(traceId) - return true - } + fun doesTransactionHaveOp(transaction: SentryTransaction, op: String): Boolean = + transaction.hasOp(op) fun doesTransactionHave( transaction: SentryTransaction, op: String, featureFlag: FeatureFlag? = null, - ): Boolean { - val matches = transaction.contexts.trace?.operation == op - if (!matches) { - println("Unable to find transaction with op $op:") - logObject(transaction) - return false - } - - val foundFlag = transaction.contexts.trace?.data?.get(featureFlag?.flag) - if (featureFlag != null && foundFlag == null) { - println("Unable to find span with feature flag ${featureFlag?.flag}:") - logObject(transaction) - return false - } - if (featureFlag != null && foundFlag != featureFlag.result) { - println("Feature flag ${featureFlag?.flag} has unexpected result ${foundFlag}:") - logObject(transaction) - return false - } - - return true - } + ): Boolean = transaction.hasOp(op, featureFlag) fun doesTransactionHaveSpanWith( transaction: SentryTransaction, op: String, featureFlag: FeatureFlag? = null, noFeatureFlags: Boolean = false, - ): Boolean { - val foundSpan = transaction.spans.firstOrNull { span -> span.op == op } - if (foundSpan == null) { - println("Unable to find span with op $op:") - logObject(transaction) - return false - } + ): Boolean = transaction.hasSpanWithOp(op, featureFlag, noFeatureFlags) - val featureFlagNames = - foundSpan.data?.keys?.filter { it.startsWith("flag.evaluation.") } ?: emptyList() - if (noFeatureFlags && featureFlagNames.isNotEmpty()) { - println("Expected 0 feature flags but found ${featureFlagNames}:") - logObject(transaction) - return false - } - - val foundFlag = foundSpan.data?.get(featureFlag?.flag) - if (featureFlag != null && foundFlag == null) { - println("Unable to find span with feature flag ${featureFlag?.flag}:") - logObject(transaction) - return false - } - if (featureFlag != null && foundFlag != featureFlag.result) { - println("Feature flag ${featureFlag?.flag} has unexpected result ${foundFlag}:") - logObject(transaction) - return false - } + fun doesEventHaveExceptionMessage(event: SentryEvent, expectedMessage: String): Boolean = + event.hasExceptionMessage(expectedMessage) - return true - } - - fun doesEventHaveExceptionMessage(event: SentryEvent, expectedMessage: String): Boolean { - val exceptions = event.exceptions - if (exceptions == null) { - println("Unable to find exceptions in event") - return false - } - - val foundException = exceptions.firstOrNull { expectedMessage == it.value } - return foundException != null - } - - fun doesEventHaveFlag(event: SentryEvent, flag: String, result: Boolean): Boolean { - val featureFlags = event.contexts.featureFlags - if (featureFlags == null) { - println("Unable to find feature flags in event:") - return false - } - val foundFlag = - featureFlags.values.firstOrNull { featureFlag -> - println("checking flag ${featureFlag.flag}:${featureFlag.result}") - featureFlag.flag == flag && featureFlag.result == result - } - return foundFlag != null - } + fun doesEventHaveFlag(event: SentryEvent, flag: String, result: Boolean): Boolean = + event.hasFlag(flag, result) fun findJar(prefix: String, inDir: String = "build/libs"): File { val buildDir = File(inDir) diff --git a/sentry-test-support/src/main/kotlin/io/sentry/test/EnvelopeMatchers.kt b/sentry-test-support/src/main/kotlin/io/sentry/test/EnvelopeMatchers.kt new file mode 100644 index 00000000000..16f7642f358 --- /dev/null +++ b/sentry-test-support/src/main/kotlin/io/sentry/test/EnvelopeMatchers.kt @@ -0,0 +1,170 @@ +package io.sentry.test + +import io.sentry.JsonSerializer +import io.sentry.ProfileChunk +import io.sentry.SentryEnvelope +import io.sentry.SentryEnvelopeItem +import io.sentry.SentryEvent +import io.sentry.SentryItemType +import io.sentry.SentryLogEvents +import io.sentry.SentryMetricsEvents +import io.sentry.SentryOptions +import io.sentry.protocol.FeatureFlag +import io.sentry.protocol.SentryTransaction +import java.io.PrintWriter + +// Matchers return false instead of throwing, so tests can pick one envelope out of many. On a +// mismatch they print what they looked at, which is otherwise hard to see from a failed wait. + +private val serializer = JsonSerializer(SentryOptions.empty()) + +fun SentryEnvelope.event(): SentryEvent? = + firstItem(SentryItemType.Event) { it.getEvent(serializer) } + +fun SentryEnvelope.transaction(): SentryTransaction? = + firstItem(SentryItemType.Transaction) { it.getTransaction(serializer) } + +fun SentryEnvelope.logs(): SentryLogEvents? = + firstItem(SentryItemType.Log) { it.getLogs(serializer) } + +fun SentryEnvelope.metrics(): SentryMetricsEvents? = + firstItem(SentryItemType.TraceMetric) { it.getMetrics(serializer) } + +fun SentryEnvelope.profileChunk(): ProfileChunk? = + firstItem(SentryItemType.ProfileChunk) { item -> + item.data.inputStream().reader().use { serializer.deserialize(it, ProfileChunk::class.java) } + } + +/** Matches the transaction's own op and, if given, a feature flag in its trace data. */ +fun SentryTransaction.hasOp(op: String, featureFlag: FeatureFlag? = null): Boolean { + val trace = contexts.trace + if (trace?.operation != op) { + return mismatch("Unable to find transaction with op $op:", this) + } + return featureFlag == null || matchesFlag(trace.data, featureFlag) +} + +fun SentryTransaction.hasTraceId(traceId: String): Boolean { + if (contexts.trace?.traceId?.toString() != traceId) { + return mismatch("Unable to find trace ID $traceId in transaction:", this) + } + return true +} + +/** + * Matches a span with [op] and, if given, a feature flag in its data. With [noFeatureFlags] the + * span must carry no flag evaluations at all. + */ +fun SentryTransaction.hasSpanWithOp( + op: String, + featureFlag: FeatureFlag? = null, + noFeatureFlags: Boolean = false, +): Boolean { + val span = + spans.firstOrNull { it.op == op } ?: return mismatch("Unable to find span with op $op:", this) + val data = span.data ?: emptyMap() + if (noFeatureFlags && data.keys.any { it.startsWith("flag.evaluation.") }) { + return mismatch("Expected no feature flags on span with op $op:", this) + } + return featureFlag == null || matchesFlag(data, featureFlag) +} + +fun SentryTransaction.hasSpanWithOpAndDescription(op: String, description: String): Boolean { + if (spans.none { it.op == op && it.description == description }) { + return mismatch( + "Unable to find span with op $op and description $description in transaction:", + this, + ) + } + return true +} + +fun SentryLogEvents.hasLogWithBody(body: String): Boolean { + if (items.none { it.body == body }) { + return mismatch("Unable to find log item with body $body in logs:", this) + } + return true +} + +fun SentryLogEvents.hasLogWithAttribute(body: String, key: String, value: Any?): Boolean { + val log = + items.firstOrNull { it.body == body } + ?: return mismatch("Unable to find log item with body $body in logs:", this) + val attribute = + log.attributes?.get(key) + ?: return mismatch("Unable to find attribute $key on log with body $body:", log) + if (attribute.value != value) { + return mismatch("Attribute $key is ${attribute.value}, expected $value on log $body:", log) + } + return true +} + +fun SentryMetricsEvents.hasMetric( + name: String, + type: String, + value: Double, + unit: String? = null, +): Boolean { + val found = items.any { + it.name == name && it.type == type && it.value == value && (unit == null || it.unit == unit) + } + if (!found) { + return mismatch("Unable to find metric $name of type $type with value $value in metrics:", this) + } + return true +} + +fun SentryMetricsEvents.hasMetricWithAttribute(name: String, key: String, value: Any?): Boolean { + val metric = + items.firstOrNull { it.name == name } + ?: return mismatch("Unable to find metric with name $name in metrics:", this) + val attribute = + metric.attributes?.get(key) + ?: return mismatch("Unable to find attribute $key on metric $name:", metric) + if (attribute.value != value) { + return mismatch( + "Attribute $key is ${attribute.value}, expected $value on metric $name:", + metric, + ) + } + return true +} + +fun SentryEvent.hasExceptionMessage(message: String): Boolean { + if (exceptions?.none { it.value == message } != false) { + return mismatch("Unable to find exception with message $message in event:", this) + } + return true +} + +fun SentryEvent.hasFlag(flag: String, result: Boolean): Boolean { + if (contexts.featureFlags?.values?.none { it.flag == flag && it.result == result } != false) { + return mismatch("Unable to find feature flag $flag=$result in event:", this) + } + return true +} + +private inline fun SentryEnvelope.firstItem( + type: SentryItemType, + read: (SentryEnvelopeItem) -> T?, +): T? = items.firstOrNull { it.header.type == type }?.let(read) + +private fun matchesFlag(data: Map?, featureFlag: FeatureFlag): Boolean { + val result = data?.get(featureFlag.flag) + if (result != featureFlag.result) { + println("Feature flag ${featureFlag.flag} is $result, expected ${featureFlag.result}") + return false + } + return true +} + +private fun mismatch(message: String, subject: Any): Boolean { + println(message) + // Flush rather than close: closing the writer would close System.out. + PrintWriter(System.out).apply { + serializer.serialize(subject, this) + println() + flush() + } + return false +} diff --git a/sentry-test-support/src/test/kotlin/io/sentry/test/EnvelopeMatchersTest.kt b/sentry-test-support/src/test/kotlin/io/sentry/test/EnvelopeMatchersTest.kt new file mode 100644 index 00000000000..68a71809e29 --- /dev/null +++ b/sentry-test-support/src/test/kotlin/io/sentry/test/EnvelopeMatchersTest.kt @@ -0,0 +1,92 @@ +package io.sentry.test + +import com.google.common.truth.Truth.assertThat +import io.sentry.Sentry +import io.sentry.protocol.FeatureFlag +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test + +class EnvelopeMatchersTest { + private val server = MockSentryServer() + + @BeforeTest + fun setup() { + initForTest { + it.dsn = server.dsn + it.tracesSampleRate = 1.0 + it.logs.isEnabled = true + } + } + + @AfterTest + fun teardown() { + Sentry.close() + server.close() + } + + @Test + fun `matches a transaction by op, trace ID and span`() { + val transaction = Sentry.startTransaction("name", "http.server") + transaction.startChild("db.query", "select 1").apply { + setData("flag.evaluation.my-flag", true) + finish() + } + transaction.finish() + Sentry.flush(5000) + + val received = server.awaitEnvelope { it.transaction() != null }.transaction()!! + + assertThat(received.hasOp("http.server")).isTrue() + assertThat(received.hasOp("other")).isFalse() + assertThat(received.hasTraceId(transaction.spanContext.traceId.toString())).isTrue() + assertThat(received.hasSpanWithOpAndDescription("db.query", "select 1")).isTrue() + assertThat(received.hasSpanWithOp("db.query", FeatureFlag("flag.evaluation.my-flag", true))) + .isTrue() + assertThat(received.hasSpanWithOp("db.query", FeatureFlag("flag.evaluation.my-flag", false))) + .isFalse() + assertThat(received.hasSpanWithOp("db.query", noFeatureFlags = true)).isFalse() + } + + @Test + fun `matches an event by exception message and feature flag`() { + Sentry.addFeatureFlag("my-flag", true) + Sentry.captureException(RuntimeException("boom")) + Sentry.flush(5000) + + val event = server.awaitEnvelope { it.event() != null }.event()!! + + assertThat(event.hasExceptionMessage("boom")).isTrue() + assertThat(event.hasExceptionMessage("other")).isFalse() + assertThat(event.hasFlag("my-flag", true)).isTrue() + assertThat(event.hasFlag("my-flag", false)).isFalse() + } + + @Test + fun `matches logs by body and attribute`() { + Sentry.setAttribute("user.type", "admin") + Sentry.logger().info("hello") + Sentry.flush(5000) + + val logs = server.awaitEnvelope { it.logs() != null }.logs()!! + + assertThat(logs.hasLogWithBody("hello")).isTrue() + assertThat(logs.hasLogWithBody("other")).isFalse() + assertThat(logs.hasLogWithAttribute("hello", "user.type", "admin")).isTrue() + assertThat(logs.hasLogWithAttribute("hello", "user.type", "guest")).isFalse() + } + + @Test + fun `matches metrics by value and attribute`() { + Sentry.setAttribute("user.type", "admin") + Sentry.metrics().gauge("my.gauge", 5.0) + Sentry.flush(5000) + + val metrics = server.awaitEnvelope { it.metrics() != null }.metrics()!! + + assertThat(metrics.hasMetric("my.gauge", "gauge", 5.0)).isTrue() + assertThat(metrics.hasMetric("my.gauge", "gauge", 6.0)).isFalse() + assertThat(metrics.hasMetricWithAttribute("my.gauge", "user.type", "admin")).isTrue() + assertThat(metrics.hasMetricWithAttribute("my.gauge", "user.type", "guest")).isFalse() + } +}