+ * This is a port of the Node agent's mongo hook. Each collection operation is
+ * recorded as two events:
+ *
+ *
a function call on the driver's collection class, with the parameters
+ * named the way the Node agent names them ({@code filter}, {@code update},
+ * {@code doc}, ...) rather than the driver's own parameter names, and
+ *
nested under it, a {@code sql_query} event with database_type
+ * {@code mongodb} whose "sql" is the normalized statement built by
+ * {@link MongoQueryShape}, so that Mongo operations land in the same digests
+ * and diffs as SQL queries.
+ *
+ *
+ *
+ * Overloads: a leading {@code ClientSession} argument is recorded as a
+ * parameter named {@code session} and a trailing result {@code Class} as
+ * {@code resultClass}; neither takes part in the statement. The remaining
+ * arguments map onto the Node names positionally.
+ *
+ *
+ * {@code find}, {@code aggregate}, {@code watch}, {@code listIndexes} and
+ * {@code distinct} return lazy iterables; their statement is recorded when the
+ * iterable is created, so options applied to the iterable afterwards
+ * ({@code sort}, {@code limit}, {@code projection}) are not part of it.
+ *
+ *
+ * The hook has no compile-time dependency on the driver. It reaches the
+ * collection's namespace and codec registry, and converts {@code Bson} values,
+ * write models and POJOs to documents, by reflection
+ * ({@link MongoDocumentConverter}). Any failure there degrades the shape to
+ * {@code ?}; it never fails the application's call.
+ *
+ *
+ * Every lookup is defensive: the agent may run against any driver version
+ * (3.x through 5.x), or against a wrapper that only implements part of the
+ * API. Anything that cannot be converted is reported as null and rendered as
+ * {@code ?}.
+ */
+final class MongoDocumentConverter implements MongoQueryShape.DocumentConverter {
+ private static final TaggedLogger logger = AppMapConfig.getLogger(null);
+
+ static final String BSON = "org.bson.conversions.Bson";
+ static final String BSON_VALUE = "org.bson.BsonValue";
+ static final String BSON_DOCUMENT = "org.bson.BsonDocument";
+ static final String BSON_WRITER = "org.bson.BsonWriter";
+ static final String BSON_DOCUMENT_WRITER = "org.bson.BsonDocumentWriter";
+ static final String CODEC_REGISTRY = "org.bson.codecs.configuration.CodecRegistry";
+ static final String ENCODER = "org.bson.codecs.Encoder";
+ static final String ENCODER_CONTEXT = "org.bson.codecs.EncoderContext";
+ static final String CLIENT_SESSION = "com.mongodb.session.ClientSession";
+ static final String WRITE_MODEL = "com.mongodb.client.model.WriteModel";
+ static final String INDEX_MODEL = "com.mongodb.client.model.IndexModel";
+ static final String MONGO_NAMESPACE = "com.mongodb.MongoNamespace";
+
+ // Whether a class is (a subtype of) a named driver type. Keyed by the class,
+ // then by the type name. Classes are held weakly so unloaded applications
+ // are not pinned.
+ private static final Map, ConcurrentHashMap> typeCache = Collections
+ .synchronizedMap(new WeakHashMap, ConcurrentHashMap>());
+
+ static boolean isA(Object value, String typeName) {
+ if (value == null) {
+ return false;
+ }
+ Class cls = value.getClass();
+ ConcurrentHashMap forClass = typeCache.get(cls);
+ if (forClass == null) {
+ forClass = new ConcurrentHashMap();
+ typeCache.put(cls, forClass);
+ }
+ Boolean known = forClass.get(typeName);
+ if (known == null) {
+ known = hasSupertype(cls, typeName);
+ forClass.put(typeName, known);
+ }
+ return known;
+ }
+
+ private static boolean hasSupertype(Class cls, String typeName) {
+ for (Class c = cls; c != null; c = c.getSuperclass()) {
+ if (c.getName().equals(typeName)) {
+ return true;
+ }
+ for (Class i : c.getInterfaces()) {
+ if (i.getName().equals(typeName) || hasSupertype(i, typeName)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ static boolean isClientSession(Object value) {
+ return isA(value, CLIENT_SESSION);
+ }
+
+ // Reflection handles for one class loader's copy of the driver.
+ private static final class Handles {
+ final Class bsonDocument;
+ final Method toBsonDocument;
+ /** The no-argument overload, available from bson 4.2 on; null before that. */
+ final Method toBsonDocumentDefault;
+ final Method registryGet;
+ final Method encode;
+ final Object encoderContext;
+ final java.lang.reflect.Constructor documentWriter;
+
+ Handles(ClassLoader loader) throws Exception {
+ Class bson = Class.forName(BSON, false, loader);
+ Class registry = Class.forName(CODEC_REGISTRY, false, loader);
+ bsonDocument = Class.forName(BSON_DOCUMENT, false, loader);
+ toBsonDocument = bson.getMethod("toBsonDocument", Class.class, registry);
+ Method noArgs = null;
+ try {
+ noArgs = bson.getMethod("toBsonDocument");
+ } catch (NoSuchMethodException e) {
+ // bson < 4.2
+ }
+ toBsonDocumentDefault = noArgs;
+ registryGet = registry.getMethod("get", Class.class);
+ Class writer = Class.forName(BSON_WRITER, false, loader);
+ Class context = Class.forName(ENCODER_CONTEXT, false, loader);
+ encode = Class.forName(ENCODER, false, loader).getMethod("encode", writer, Object.class, context);
+ Object builder = context.getMethod("builder").invoke(null);
+ encoderContext = builder.getClass().getMethod("build").invoke(builder);
+ documentWriter = Class.forName(BSON_DOCUMENT_WRITER, false, loader).getConstructor(bsonDocument);
+ }
+ }
+
+ private static final Map handlesByLoader = Collections
+ .synchronizedMap(new WeakHashMap());
+ private static final Handles NO_HANDLES = null;
+
+ private static Handles handles(ClassLoader loader) {
+ if (loader == null) {
+ loader = ClassLoader.getSystemClassLoader();
+ }
+ synchronized (handlesByLoader) {
+ if (handlesByLoader.containsKey(loader)) {
+ return handlesByLoader.get(loader);
+ }
+ Handles h;
+ try {
+ h = new Handles(loader);
+ } catch (Throwable e) {
+ logger.debug(e, "bson classes not available from {}", loader);
+ h = NO_HANDLES;
+ }
+ handlesByLoader.put(loader, h);
+ return h;
+ }
+ }
+
+ // Accessors on the collection object itself, cached per collection class.
+ private static final Map, Method[]> collectionAccessors = Collections
+ .synchronizedMap(new WeakHashMap, Method[]>());
+
+ private static Method[] accessors(Object collection) {
+ Class cls = collection.getClass();
+ Method[] found = collectionAccessors.get(cls);
+ if (found == null) {
+ found = new Method[2];
+ try {
+ found[0] = cls.getMethod("getNamespace");
+ found[0].setAccessible(true);
+ } catch (Throwable e) {
+ logger.debug(e, "{} has no getNamespace()", cls.getName());
+ }
+ try {
+ found[1] = cls.getMethod("getCodecRegistry");
+ found[1].setAccessible(true);
+ } catch (Throwable e) {
+ logger.debug(e, "{} has no getCodecRegistry()", cls.getName());
+ }
+ collectionAccessors.put(cls, found);
+ }
+ return found;
+ }
+
+ /** The collection name, or null when it cannot be determined. */
+ static String collectionName(Object collection) {
+ if (collection == null) {
+ return null;
+ }
+ Method getNamespace = accessors(collection)[0];
+ if (getNamespace == null) {
+ return null;
+ }
+ try {
+ Object namespace = getNamespace.invoke(collection);
+ if (namespace == null) {
+ return null;
+ }
+ Object name = namespace.getClass().getMethod("getCollectionName").invoke(namespace);
+ return name == null ? null : name.toString();
+ } catch (Throwable e) {
+ logger.debug(e, "failed to get collection name");
+ return null;
+ }
+ }
+
+ static MongoQueryShape.DocumentConverter forCollection(Object collection) {
+ Object registry = null;
+ ClassLoader loader = collection == null ? null : collection.getClass().getClassLoader();
+ if (collection != null) {
+ Method getCodecRegistry = accessors(collection)[1];
+ if (getCodecRegistry != null) {
+ try {
+ registry = getCodecRegistry.invoke(collection);
+ } catch (Throwable e) {
+ logger.debug(e, "failed to get codec registry");
+ }
+ }
+ }
+ return new MongoDocumentConverter(loader, registry);
+ }
+
+ private final Handles handles;
+ private final Object registry;
+
+ MongoDocumentConverter(ClassLoader loader, Object registry) {
+ this.handles = handles(loader);
+ this.registry = registry;
+ }
+
+ @Override
+ public Object convert(Object value, boolean documentPosition) {
+ if (value == null) {
+ return null;
+ }
+ try {
+ if (isA(value, BSON)) {
+ return toBsonDocument(value);
+ }
+ if (isA(value, BSON_VALUE)) {
+ // A scalar BsonValue (BsonDocument and BsonArray are a Map and a List,
+ // and never get here).
+ return null;
+ }
+ if (isA(value, WRITE_MODEL)) {
+ return writeModel(value);
+ }
+ if (isA(value, INDEX_MODEL)) {
+ Map shape = new LinkedHashMap();
+ shape.put("key", getter(value, "getKeys"));
+ return shape;
+ }
+ if (isA(value, MONGO_NAMESPACE)) {
+ Object fullName = getter(value, "getFullName");
+ return fullName == null ? null : fullName.toString();
+ }
+ if (documentPosition && !isJdkType(value)) {
+ return encode(value);
+ }
+ } catch (Throwable e) {
+ logger.debug(e, "failed to convert {} to a document", value.getClass().getName());
+ }
+ return null;
+ }
+
+ private static boolean isJdkType(Object value) {
+ String name = value.getClass().getName();
+ return name.startsWith("java.") || name.startsWith("javax.");
+ }
+
+ private Object toBsonDocument(Object bson) throws Exception {
+ if (handles == null) {
+ return null;
+ }
+ if (registry != null) {
+ return handles.toBsonDocument.invoke(bson, handles.bsonDocument, registry);
+ }
+ if (handles.toBsonDocumentDefault != null) {
+ return handles.toBsonDocumentDefault.invoke(bson);
+ }
+ return null;
+ }
+
+ /**
+ * Encodes an arbitrary document object (a POJO, a Kotlin data class, ...) with
+ * the collection's codec registry, the way the driver will when it sends it.
+ * Returns null when the registry has no codec for it.
+ */
+ private Object encode(Object document) throws Exception {
+ if (handles == null || registry == null) {
+ return null;
+ }
+ Object codec;
+ try {
+ codec = handles.registryGet.invoke(registry, document.getClass());
+ } catch (Throwable e) {
+ // CodecConfigurationException: no codec for this class
+ return null;
+ }
+ if (codec == null) {
+ return null;
+ }
+ Object target = handles.bsonDocument.getConstructor().newInstance();
+ Object writer = handles.documentWriter.newInstance(target);
+ handles.encode.invoke(codec, writer, document, handles.encoderContext);
+ return target;
+ }
+
+ // Renders a WriteModel the way the Node driver's bulkWrite operations look:
+ // { "insertOne": { "document": ... } }, { "updateOne": { "filter": ...,
+ // "update": ... } }, and so on.
+ private Object writeModel(Object model) {
+ String kind = model.getClass().getSimpleName();
+ Map body = new LinkedHashMap();
+ String key;
+ switch (kind) {
+ case "InsertOneModel":
+ key = "insertOne";
+ body.put("document", document(getter(model, "getDocument")));
+ break;
+ case "UpdateOneModel":
+ case "UpdateManyModel":
+ key = kind.equals("UpdateOneModel") ? "updateOne" : "updateMany";
+ body.put("filter", getter(model, "getFilter"));
+ Object update = getter(model, "getUpdate");
+ if (update == null) {
+ update = getter(model, "getUpdatePipeline");
+ }
+ body.put("update", update);
+ break;
+ case "ReplaceOneModel":
+ key = "replaceOne";
+ body.put("filter", getter(model, "getFilter"));
+ body.put("replacement", document(getter(model, "getReplacement")));
+ break;
+ case "DeleteOneModel":
+ case "DeleteManyModel":
+ key = kind.equals("DeleteOneModel") ? "deleteOne" : "deleteMany";
+ body.put("filter", getter(model, "getFilter"));
+ break;
+ default:
+ return null;
+ }
+ Map shape = new LinkedHashMap();
+ shape.put(key, body);
+ return shape;
+ }
+
+ // A document nested in a write model sits in a document position, but the
+ // shape renderer only knows that for top-level arguments. Convert it here.
+ private Object document(Object value) {
+ if (value == null || value instanceof Map || MongoQueryShape.isSequence(value)) {
+ return value;
+ }
+ Object converted = convert(value, true);
+ return converted == null ? value : converted;
+ }
+
+ private static Object getter(Object target, String name) {
+ if (target == null) {
+ return null;
+ }
+ try {
+ Method m = target.getClass().getMethod(name);
+ m.setAccessible(true);
+ return m.invoke(target);
+ } catch (Throwable e) {
+ logger.debug(e, "{}.{}() failed", target.getClass().getName(), name);
+ return null;
+ }
+ }
+}
diff --git a/agent/src/main/java/com/appland/appmap/process/hooks/MongoQueryShape.java b/agent/src/main/java/com/appland/appmap/process/hooks/MongoQueryShape.java
new file mode 100644
index 00000000..2816fb81
--- /dev/null
+++ b/agent/src/main/java/com/appland/appmap/process/hooks/MongoQueryShape.java
@@ -0,0 +1,345 @@
+package com.appland.appmap.process.hooks;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * Renders a MongoDB collection operation as a query statement with a
+ * normalized argument shape, so that it can be recorded as a {@code sql_query}
+ * event (database_type "mongodb") and compared across recordings the way SQL
+ * is.
+ *
+ *
+ * The same rules are implemented by the Node agent
+ * ({@code src/hooks/mongoQuery.ts} in appmap-node). Keep them in sync.
+ *
+ *
+ * Statement form: {@code db..(, , ...)}
+ *
+ *
The collection is written {@code db.name} when the name is a plain
+ * identifier path, and {@code db.getCollection("name")} otherwise.
+ *
Arguments are rendered in the order of the driver method's parameters.
+ * Trailing null arguments are omitted.
+ *
Documents (filters, updates, replacements, inserted documents, index
+ * specs) keep their keys, in order, and every leaf value becomes {@code ?}.
+ * Keys are written as JSON strings.
+ *
Arrays inside documents, and top-level lists of documents (insertMany,
+ * bulkWrite, createIndexes), keep only the distinct element shapes, in order
+ * of first appearance. {@code {"$in": [1, 2, 3]}} becomes
+ * {@code {"$in": [?]}}, and a thousand inserted documents of the same shape
+ * become one.
+ *
Aggregation pipelines (aggregate, watch, and update pipelines) keep
+ * every stage in order, because stage order and repetition are part of the
+ * query.
+ *
Name-like arguments (a distinct field, an index name, a new collection
+ * name) are kept verbatim as JSON strings; they identify the query the same
+ * way a table or column name does.
+ *
Everything that is not a Map, an Iterable, an array, or convertible to
+ * a document by the {@link DocumentConverter} (BSON values such as ObjectId
+ * and Decimal128, primitives, Dates, null) is a leaf and becomes
+ * {@code ?}.
+ *
Options objects ({@code UpdateOptions}, {@code IndexOptions}, ...) are
+ * opaque in Java and become {@code ?}. The Node agent renders the keys of its
+ * options document instead.
+ *
Nesting deeper than {@link #MAX_DEPTH}, and cyclic references, become
+ * {@code ?}. Only the first {@link #MAX_ARRAY_ELEMENTS} elements of an array
+ * are examined.
+ *
+ */
+public final class MongoQueryShape {
+ public static final String DATABASE_TYPE = "mongodb";
+ public static final int MAX_DEPTH = 32;
+ public static final int MAX_ARRAY_ELEMENTS = 1000;
+
+ static final String PLACEHOLDER = "?";
+
+ /**
+ * Turns driver-specific objects into documents the shape renderer can walk.
+ * Implementations reach into the driver by reflection; this class has no
+ * dependency on it.
+ */
+ public interface DocumentConverter {
+ /**
+ * @param value a non-null value that is not a Map, Iterable or array
+ * @param documentPosition true when the value sits where the driver expects
+ * a document (an inserted document, a replacement, a pipeline
+ * stage), so that an arbitrary object may be encoded with the
+ * collection's codec
+ * @return a Map, Iterable or array to walk instead of {@code value}, a
+ * String to render verbatim, or null when the value is a leaf
+ */
+ Object convert(Object value, boolean documentPosition);
+ }
+
+ static final DocumentConverter NO_CONVERSION = new DocumentConverter() {
+ @Override
+ public Object convert(Object value, boolean documentPosition) {
+ return null;
+ }
+ };
+
+ enum ArgKind {
+ DOCUMENT, PIPELINE, NAME, OPTIONS
+ }
+
+ static ArgKind argKind(String name) {
+ switch (name) {
+ case "pipeline":
+ return ArgKind.PIPELINE;
+ case "key":
+ case "newName":
+ case "indexName":
+ case "indexes":
+ return ArgKind.NAME;
+ case "options":
+ return ArgKind.OPTIONS;
+ default:
+ return ArgKind.DOCUMENT;
+ }
+ }
+
+ private MongoQueryShape() {
+ }
+
+ /**
+ * Formats a collection method call as a normalized statement. Never throws: if
+ * the arguments cannot be inspected the statement is rendered with a single
+ * {@code ?} in place of the argument list.
+ */
+ public static String formatStatement(String collection, String method, String[] argNames, Object[] args,
+ DocumentConverter converter) {
+ String prefix = formatCollection(collection) + "." + method;
+ try {
+ return prefix + "(" + formatArgs(argNames, args, converter) + ")";
+ } catch (Throwable e) {
+ return prefix + "(" + PLACEHOLDER + ")";
+ }
+ }
+
+ private static final Pattern IDENTIFIER_PATH = Pattern.compile("[A-Za-z_$][\\w$]*(\\.[A-Za-z_$][\\w$]*)*");
+
+ public static String formatCollection(String name) {
+ if (name == null) {
+ return "db.getCollection(" + PLACEHOLDER + ")";
+ }
+ if (IDENTIFIER_PATH.matcher(name).matches()) {
+ return "db." + name;
+ }
+ return "db.getCollection(" + quote(name) + ")";
+ }
+
+ private static String formatArgs(String[] argNames, Object[] args, DocumentConverter converter) {
+ int last = Math.min(argNames.length, args == null ? 0 : args.length);
+ while (last > 0 && args[last - 1] == null) {
+ last--;
+ }
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < last; i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+ sb.append(formatArg(argNames[i], args[i], converter));
+ }
+ return sb.toString();
+ }
+
+ private static String formatArg(String name, Object value, DocumentConverter converter) {
+ switch (argKind(name)) {
+ case NAME:
+ if (value instanceof CharSequence) {
+ return quote(value.toString());
+ }
+ if (value instanceof Iterable && allStrings((Iterable) value)) {
+ StringBuilder sb = new StringBuilder("[");
+ boolean first = true;
+ for (Object v : (Iterable) value) {
+ if (!first) {
+ sb.append(", ");
+ }
+ first = false;
+ sb.append(quote(v.toString()));
+ }
+ return sb.append("]").toString();
+ }
+ return shape(value, false, false, converter);
+ case PIPELINE:
+ return shape(value, true, true, converter);
+ case OPTIONS:
+ return shape(value, false, false, converter);
+ case DOCUMENT:
+ default:
+ // An update can be a pipeline (a list of stages) instead of a document.
+ boolean ordered = "update".equals(name) && isSequence(value);
+ return shape(value, ordered, true, converter);
+ }
+ }
+
+ private static boolean allStrings(Iterable values) {
+ for (Object v : values) {
+ if (!(v instanceof CharSequence)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Renders the shape of a value: keys kept, leaves replaced with {@code ?}.
+ *
+ * @param ordered when true, a top-level array keeps all of its elements in
+ * order (a pipeline); otherwise distinct element shapes are kept
+ * @param documentPosition whether the value (or the elements of a top-level
+ * array) sit where the driver expects a document
+ */
+ public static String shape(Object value, boolean ordered, boolean documentPosition, DocumentConverter converter) {
+ StringBuilder sb = new StringBuilder();
+ shapeOf(value, ordered, documentPosition, 0, new IdentityHashMap(), converter, sb);
+ return sb.toString();
+ }
+
+ private static void shapeOf(Object value, boolean ordered, boolean documentPosition, int depth,
+ IdentityHashMap ancestors, DocumentConverter converter, StringBuilder sb) {
+ if (depth > MAX_DEPTH || value == null) {
+ sb.append(PLACEHOLDER);
+ return;
+ }
+
+ if (!(value instanceof Map) && !isSequence(value)) {
+ Object converted = converter.convert(value, documentPosition);
+ if (converted == null) {
+ sb.append(PLACEHOLDER);
+ return;
+ }
+ if (converted instanceof CharSequence) {
+ sb.append(quote(converted.toString()));
+ return;
+ }
+ value = converted;
+ }
+
+ if (ancestors.containsKey(value)) {
+ sb.append(PLACEHOLDER);
+ return;
+ }
+ ancestors.put(value, Boolean.TRUE);
+ try {
+ if (value instanceof Map) {
+ shapeOfMap((Map) value, depth, ancestors, converter, sb);
+ } else if (isSequence(value)) {
+ shapeOfSequence(value, ordered, documentPosition, depth, ancestors, converter, sb);
+ } else {
+ sb.append(PLACEHOLDER);
+ }
+ } finally {
+ ancestors.remove(value);
+ }
+ }
+
+ private static void shapeOfMap(Map map, int depth, IdentityHashMap ancestors,
+ DocumentConverter converter, StringBuilder sb) {
+ sb.append('{');
+ boolean first = true;
+ for (Map.Entry entry : map.entrySet()) {
+ if (!first) {
+ sb.append(", ");
+ }
+ first = false;
+ sb.append(quote(String.valueOf(entry.getKey()))).append(": ");
+ shapeOf(entry.getValue(), false, false, depth + 1, ancestors, converter, sb);
+ }
+ sb.append('}');
+ }
+
+ private static void shapeOfSequence(Object value, boolean ordered, boolean documentPosition, int depth,
+ IdentityHashMap ancestors, DocumentConverter converter, StringBuilder sb) {
+ List parts = new ArrayList();
+ Set seen = ordered ? null : new HashSet();
+ Iterator it = iterator(value);
+ for (int i = 0; i < MAX_ARRAY_ELEMENTS && it.hasNext(); i++) {
+ StringBuilder part = new StringBuilder();
+ shapeOf(it.next(), false, documentPosition, depth + 1, ancestors, converter, part);
+ String rendered = part.toString();
+ if (ordered || seen.add(rendered)) {
+ parts.add(rendered);
+ }
+ }
+ sb.append('[');
+ for (int i = 0; i < parts.size(); i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+ sb.append(parts.get(i));
+ }
+ sb.append(']');
+ }
+
+ static boolean isSequence(Object value) {
+ return value instanceof Iterable || (value != null && value.getClass().isArray()
+ && !value.getClass().getComponentType().isPrimitive());
+ }
+
+ private static Iterator iterator(Object value) {
+ if (value instanceof Iterable) {
+ return ((Iterable) value).iterator();
+ }
+ final Object[] array = (Object[]) value;
+ return new Iterator