[#53] Track SerializeReference instances in analyze

Analyzer.Tests/FileDetectionTests.cs

@@ -512,10 +512,10 @@ public void TryParseMetadata_V22PrefabWithSerializedReference_ReturnsExpectedObj
         for (int i = 0; i < expected.Length; i++)
         {
             var obj = metadata.ObjectList[i];
-            Assert.That(obj.Id,     Is.EqualTo(expected[i].Id),     $"ObjectList[{i}].Id");
+            Assert.That(obj.Id, Is.EqualTo(expected[i].Id), $"ObjectList[{i}].Id");
             Assert.That(obj.TypeId, Is.EqualTo(expected[i].TypeId), $"ObjectList[{i}].TypeId");
             Assert.That(obj.Offset, Is.EqualTo(expected[i].Offset), $"ObjectList[{i}].Offset");
-            Assert.That(obj.Size,   Is.EqualTo(expected[i].Size),   $"ObjectList[{i}].Size");
+            Assert.That(obj.Size, Is.EqualTo(expected[i].Size), $"ObjectList[{i}].Size");
         }
     }
 

Analyzer/PPtrAndCrcProcessor.cs

@@ -15,6 +15,8 @@ namespace UnityDataTools.Analyzer;
 //     content fingerprint used to detect whether two objects are identical.
 //     NOTE: references contribute their resolved analyzer object id (see ExtractPPtr), so the CRC
 //     is only comparable within a single analyze database, not between separate runs - see issue #74.
+// It also reports each [SerializeReference] instance it walks through, since only this walk
+// reaches them.
 // CRC computation can be disabled (skipCrc) while still extracting references.
 public class PPtrAndCrcProcessor : IDisposable
 {
@@ -28,6 +30,11 @@ public class PPtrAndCrcProcessor : IDisposable
     // caller folds into the CRC.
     public delegate int CallbackDelegate(long objectId, int fileId, long pathId, string propertyPath, string propertyType);
 
+    // Invoked for each [SerializeReference] instance held by the object. `size` is the byte length of
+    // the instance's data, excluding the type name that precedes it.
+    public delegate void ManagedReferenceCallbackDelegate(long objectId, long rid, string className,
+        string namespaceName, string assemblyName, long size);
+
     // Content-addressed stream paths (new ContentDirectory build output) look like
     // "cah:/<hash>". The hash already identifies the content, so the path itself is
     // folded into the CRC instead of opening the (differently named) resource file.
@@ -40,6 +47,7 @@ public class PPtrAndCrcProcessor : IDisposable
     private string m_Folder;                     // directory of the serialized file; used to find companion resource files
     private bool m_SkipCrc;                      // when true, skip CRC computation (references are still extracted)
     private CallbackDelegate m_Callback;         // invoked for each PPtr; returns the referenced object's id
+    private ManagedReferenceCallbackDelegate m_ManagedReferenceCallback;
 
     // Readers for external resource (.resS/.resource) files, opened on demand, reused across
     // objects, and disposed in Dispose().
@@ -62,18 +70,35 @@ public class PPtrAndCrcProcessor : IDisposable
     // skipCrc:        when true, the tree is still walked to emit references but no CRC is computed.
     // callback:       called for every PPtr found; its return value (the referenced object's id) is
     //                 folded into the CRC.
+    // managedReferenceCallback:
+    //                 called for every [SerializeReference] instance found.
     public PPtrAndCrcProcessor(
         SerializedFile serializedFile,
         UnityFileReader reader,
         string folder,
         bool skipCrc,
-        CallbackDelegate callback)
+        CallbackDelegate callback,
+        ManagedReferenceCallbackDelegate managedReferenceCallback)
     {
         m_SerializedFile = serializedFile;
         m_Reader = reader;
         m_Folder = folder;
         m_SkipCrc = skipCrc;
         m_Callback = callback;
+        m_ManagedReferenceCallback = managedReferenceCallback;
+    }
+
+    // True when objects of this type carry a [SerializeReference] registry, in either its node-described
+    // form (versions 1 and 2) or as a frame ahead of the field flagged HasSerializedRefs (version 3).
+    public static bool HasManagedReferenceRegistry(TypeTreeNode root)
+    {
+        foreach (var child in root.Children)
+        {
+            if (child.IsManagedReferenceRegistry || child.HasSerializedRefs)
+                return true;
+        }
+
+        return false;
     }
 
     public void Dispose()
@@ -456,8 +481,11 @@ private void ProcessRefTypeData(long rid, string className, string namespaceName
         m_StringBuilder.Append("rid(");
         m_StringBuilder.Append(rid);
         m_StringBuilder.Append(").data");
+        var dataStart = m_Offset;
         ProcessNode(refTypeTypeTree, true);
         m_StringBuilder.Remove(pathLength, m_StringBuilder.Length - pathLength);
+
+        m_ManagedReferenceCallback(m_ObjectId, rid, className, namespaceName, assemblyName, m_Offset - dataStart);
     }
 
     private void ExtractPPtr(string referencedType)

Analyzer/Resources/Finalize.sql

@@ -1,2 +1,3 @@
 CREATE INDEX refs_object_index ON refs(object);
 CREATE INDEX refs_referenced_object_index ON refs(referenced_object);
+CREATE INDEX managed_references_object_index ON managed_references(object);

Analyzer/Resources/Init.sql

@@ -75,6 +75,18 @@ CREATE TABLE IF NOT EXISTS dangling_refs
     PRIMARY KEY (id)
 );
 
+CREATE TABLE IF NOT EXISTS managed_references
+(
+    -- One [SerializeReference] instance held by a MonoBehaviour / ScriptableObject. Populated even
+    -- with --skip-references and --skip-crc. Null references have no row.
+    object INTEGER,          -- objects.id of the MonoBehaviour that holds the instance
+    rid INTEGER,             -- managed reference id; the entry position in registry version 1 files
+    class_name TEXT,         -- a nested class is written Outer/Inner
+    namespace TEXT,
+    assembly_name TEXT,
+    size INTEGER             -- bytes of the instance's serialized data, excluding its type name
+);
+
 CREATE VIEW refs_view AS
 -- refs with the property_path and property_type ids resolved to their strings.
 SELECT r.object, r.referenced_object, pn.name AS property_path, pt.name AS property_type
@@ -115,6 +127,35 @@ INNER JOIN types t ON o.type = t.id
 INNER JOIN serialized_files sf ON o.serialized_file = sf.id
 LEFT JOIN archives ab ON sf.archive = ab.id;
 
+CREATE VIEW managed_reference_view AS
+-- Each [SerializeReference] instance with the MonoBehaviour that holds it.
+SELECT
+    o.id,
+    o.object_id,
+    o.name,
+    o.archive,
+    o.serialized_file,
+    m.class_name,
+    m.namespace,
+    m.assembly_name,
+    m.rid,
+    m.size
+FROM managed_references m
+INNER JOIN object_view o ON m.object = o.id;
+
+CREATE VIEW managed_reference_stats_view AS
+-- Each distinct [SerializeReference] type: instance count, holding objects and total size.
+SELECT
+    class_name,
+    namespace,
+    assembly_name,
+    COUNT(*) AS instances,
+    COUNT(DISTINCT object) AS objects,
+    SUM(size) AS total_size
+FROM managed_references
+GROUP BY class_name, namespace, assembly_name
+ORDER BY instances DESC, class_name;
+
 CREATE VIEW view_breakdown_by_type AS
 -- Object count and total size per type, largest first.
 SELECT *,
@@ -168,7 +209,7 @@ WHERE m.type = 'Material';
 
 INSERT INTO types (id, name) VALUES (-1, 'Scene');
 
-PRAGMA user_version = 9;
+PRAGMA user_version = 10;
 
 PRAGMA synchronous = OFF;
 PRAGMA journal_mode = MEMORY;

Analyzer/SQLite/Commands/SerializedFile/AddManagedReference.cs

@@ -0,0 +1,24 @@
+using System.Collections.Generic;
+using Microsoft.Data.Sqlite;
+using UnityDataTools.Analyzer.SQLite.Commands;
+
+namespace UnityDataTools.Analyzer.SQLite.Commands.SerializedFile
+{
+    // Table definition: Analyzer/Resources/Init.sql
+    internal class AddManagedReference : AbstractCommand
+    {
+        protected override string TableName => "managed_references";
+
+        protected override string DDLSource => null;
+
+        protected override Dictionary<string, SqliteType> Fields => new()
+        {
+            { "object", SqliteType.Integer },
+            { "rid", SqliteType.Integer },
+            { "class_name", SqliteType.Text },
+            { "namespace", SqliteType.Text },
+            { "assembly_name", SqliteType.Text },
+            { "size", SqliteType.Integer }
+        };
+    }
+}

Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs

@@ -95,6 +95,7 @@ public class SerializedFileSQLiteWriter : IDisposable
     private AddType m_AddTypeCommand = new AddType();
     private AddPreloadDependency m_InsertDepCommand = new AddPreloadDependency();
     private AddDanglingRef m_AddDanglingRefCommand = new AddDanglingRef();
+    private AddManagedReference m_AddManagedReferenceCommand = new AddManagedReference();
 
     private bool m_Initialized;
     private SqliteConnection m_Database;
@@ -137,6 +138,7 @@ private void CreateSQLiteCommands()
         m_AddTypeCommand.CreateCommand(m_Database);
         m_InsertDepCommand.CreateCommand(m_Database);
         m_AddDanglingRefCommand.CreateCommand(m_Database);
+        m_AddManagedReferenceCommand.CreateCommand(m_Database);
 
         m_LastId = m_Database.CreateCommand();
         m_LastId.CommandText = "SELECT last_insert_rowid()";
@@ -213,7 +215,7 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
 
         using var sf = UnityFileSystem.OpenSerializedFile(fullPath);
         using var reader = new UnityFileReader(fullPath, 64 * 1024 * 1024);
-        using var pptrReader = new PPtrAndCrcProcessor(sf, reader, containingFolder, m_SkipCrc, AddReference);
+        using var pptrReader = new PPtrAndCrcProcessor(sf, reader, containingFolder, m_SkipCrc, AddReference, AddManagedReference);
         int serializedFileId = m_SerializedFileIdProvider.GetId(
             ContentFileDependencyMap.NormalizeFileName(Path.GetFileName(fullPath)));
         int sceneId = -1;
@@ -382,10 +384,10 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
                 }
                 m_AddObjectCommand.SetValue("game_object", gameObject);
 
-                // The walk both extracts references and accumulates the CRC, so it is needed
-                // unless both are disabled. When CRC is on but references are off, the walk
-                // still resolves referenced object ids (AddReference skips the insert).
-                if (!m_SkipReferences || !m_SkipCrc)
+                // The walk extracts references, accumulates the CRC and records [SerializeReference]
+                // instances, which are recorded whatever the skip options. When references are off,
+                // the walk still resolves referenced object ids (AddReference skips the insert).
+                if (!m_SkipReferences || !m_SkipCrc || PPtrAndCrcProcessor.HasManagedReferenceRegistry(root))
                 {
                     crc32 = pptrReader.Process(currentObjectId, offset, obj.Size, root);
                 }
@@ -517,6 +519,20 @@ private int AddReference(long objectId, int fileId, long pathId, string property
         return referencedObjectId;
     }
 
+    // Callback from PPtrAndCrcProcessor for each [SerializeReference] instance in the SerializedFile
+    private void AddManagedReference(long objectId, long rid, string className, string namespaceName,
+        string assemblyName, long size)
+    {
+        m_AddManagedReferenceCommand.SetTransaction(m_CurrentTransaction);
+        m_AddManagedReferenceCommand.SetValue("object", objectId);
+        m_AddManagedReferenceCommand.SetValue("rid", rid);
+        m_AddManagedReferenceCommand.SetValue("class_name", className);
+        m_AddManagedReferenceCommand.SetValue("namespace", namespaceName);
+        m_AddManagedReferenceCommand.SetValue("assembly_name", assemblyName);
+        m_AddManagedReferenceCommand.SetValue("size", size);
+        m_AddManagedReferenceCommand.ExecuteNonQuery();
+    }
+
     // Resolve a property path/type string to its id, writing the lookup row the first time the
     // string is seen. Called within the current transaction (references are being extracted).
     private int GetPropertyPathId(string propertyPath)
@@ -562,6 +578,7 @@ public void Dispose()
         m_AddTypeCommand.Dispose();
         m_InsertDepCommand.Dispose();
         m_AddDanglingRefCommand.Dispose();
+        m_AddManagedReferenceCommand.Dispose();
 
         m_LastId.Dispose();
     }

Documentation/analyze-examples.md

@@ -156,6 +156,42 @@ WHERE mb.type = 'MonoBehaviour'
   AND ms.namespace = 'UnityEngine.U2D.Animation';
 ```
 
+## Example: Finding SerializeReference types
+
+A field marked with `[SerializeReference]` stores a C# object inside the MonoBehaviour or
+ScriptableObject that holds the field. There is no MonoScript for the object's type. Instead, the
+type's class, namespace and assembly are written into every object that holds an instance. This makes
+the types hard to find in built content, so analyze collects every instance into the
+[`managed_references`](analyzer-schema.md#managed_references) table.
+
+For example, before you rename a class, you can check that no content built with the old name remains:
+
+```
+SELECT archive, serialized_file, name, object_id, class_name, namespace, assembly_name, rid
+FROM managed_reference_view
+WHERE class_name = 'MyOldClassName';
+```
+
+A nested class is recorded as `Outer/Inner`, so use `LIKE '%/MyOldClassName'` to find it without
+the outer class name.
+
+To see every type used, how often, and how many bytes its instances take:
+
+```
+SELECT * FROM managed_reference_stats_view;
+```
+
+`managed_reference_view` shows the name of the holding object, but not its own C# class. The C#
+class comes from `script_object_view`, which is populated only when analyze runs without
+`--skip-references`. Join the two views on `id` to see both:
+
+```
+SELECT so.class_name AS script_class, so.name, mr.class_name, mr.namespace, mr.rid, mr.size
+FROM managed_reference_view mr
+INNER JOIN script_object_view so ON mr.id = so.id
+ORDER BY so.class_name, mr.class_name;
+```
+
 ## Example: Quick summary for individual AssetBundles
 
 Often Analyze is used for an entire build output, so that you can view information about the build output as a whole.

Documentation/analyzer-schema.md

@@ -26,7 +26,7 @@ Parts of the schema are documented on their own pages:
 * **Ids are analyzer-assigned.** `objects.id` and the ids that reference it exist only in the
   database. The Unity object id is `objects.object_id`.
 * **Two options change what gets populated.** `--skip-references` leaves `refs`, `dangling_refs` and
-  `script_object_view` empty. `--skip-crc` sets every `objects.crc32` to 0, which makes
+  `script_object_view` empty (`managed_references` is still populated). `--skip-crc` sets every `objects.crc32` to 0, which makes
   `view_potential_duplicates` report many false positives.
 
 ## At a glance
@@ -56,7 +56,7 @@ Core views:
 | [`view_material_shader_refs`](#view_material_shader_refs-and-view_material_texture_refs) | each Material and its Shader |
 | [`view_material_texture_refs`](#view_material_shader_refs-and-view_material_texture_refs) | each Material and its Textures |
 
-AssetBundle and MonoScript:
+AssetBundle and scripting:
 
 | Name | Purpose |
 |---|---|
@@ -67,6 +67,9 @@ AssetBundle and MonoScript:
 | [`monoscripts`](#monoscripts) | C# class behind each MonoBehaviour / ScriptableObject |
 | [`monoscripts_view`](#monoscripts_view) | MonoScripts with their containing file |
 | [`script_object_view`](#script_object_view) | MonoBehaviours and ScriptableObjects with their C# type |
+| [`managed_references`](#managed_references) | `[SerializeReference]` instance held by a MonoBehaviour / ScriptableObject |
+| [`managed_reference_view`](#managed_reference_view) | those instances with the object that holds them |
+| [`managed_reference_stats_view`](#managed_reference_stats_view) | instance count and total size per `[SerializeReference]` type |
 
 Type-specific views, each `object_view` plus extra columns:
 
@@ -421,6 +424,46 @@ with `--skip-references`.
 
 ---
 
+# SerializeReference tables and views
+
+A field marked with [`[SerializeReference]`](https://docs.unity3d.com/6000.6/Documentation/ScriptReference/SerializeReference.html)
+holds a C# object (a "managed reference") that is stored inside the MonoBehaviour or ScriptableObject
+itself. Its type does not have a MonoScript. Instead, each MonoBehaviour stores the class, namespace and
+assembly of each instance in its own managed reference registry. These tables collect those instances
+from the whole build, so you can see which types are used and where. For example, you can check that a
+class you want to rename is not used anywhere.
+
+The data is the same for AssetBundles, Player builds and ContentDirectory builds, and for every
+registry format, including the one introduced in Unity 6.7. It does not depend on the `refs` table, so
+it is populated even with `--skip-references --skip-crc`.
+
+## managed_references
+
+One row per `[SerializeReference]` instance. Each instance is stored once in its MonoBehaviour's
+registry, however many fields point at it, so an instance that is shared or nested inside another
+instance has one row. Null references have no row.
+
+| Column | Type | Description |
+|---|---|---|
+| `object` | INTEGER | [`objects.id`](#objects) of the MonoBehaviour that holds the instance. |
+| `rid` | INTEGER | The managed reference id, which the fields pointing at the instance store. Unique only within its MonoBehaviour. In files built with Unity 2020 or earlier (registry version 1) it is the entry's position in the registry. |
+| `class_name` | TEXT | The instance's concrete class. A nested class is written `Outer/Inner`. |
+| `namespace` | TEXT | C# namespace; empty for the global namespace. |
+| `assembly_name` | TEXT | The assembly, e.g. `Assembly-CSharp`. |
+| `size` | INTEGER | Bytes of the instance's serialized data, excluding its type name. It is part of the holding object's `size`. |
+
+## managed_reference_view
+
+`managed_references` with the holding MonoBehaviour's `id`, `object_id`, `name`, `archive` and
+`serialized_file` from [`object_view`](#object_view).
+
+## managed_reference_stats_view
+
+One row per distinct `[SerializeReference]` type, with `instances` (row count), `objects` (distinct
+holding MonoBehaviours) and `total_size`, most used first.
+
+---
+
 # Type-specific views
 
 Each of these has the same columns as [`object_view`](#object_view) plus the ones listed.
@@ -553,6 +596,7 @@ Any schema change - a new or changed table, view or column - must bump the pragm
 | 7 | Unity 6.6 `build_reports` columns and `build_report_content_*` tables ([#107](https://github.com/Unity-Technologies/UnityDataTools/issues/107)); `asset_name` / `asset_extension` columns on `build_report_source_assets` ([#110](https://github.com/Unity-Technologies/UnityDataTools/issues/110)) |
 | 8 | `archives.name` is the path relative to the scanned directory, not the bare file name ([#149](https://github.com/Unity-Technologies/UnityDataTools/issues/149)) |
 | 9 | `content_layout*` tables track the version 3 layout schema of Unity 6.7: loadables keyed by `loadable_index`, `stable_id` / `artifact_index` columns replace `cfid` / `content_hash`, `is_root_asset` records the root position, and the v2-only columns (`asset_path`, `source_lfid`) exist only in databases imported from a version 2 layout ([#131](https://github.com/Unity-Technologies/UnityDataTools/issues/131)) |
+| 10 | Added the `managed_references` table, `managed_reference_view` and `managed_reference_stats_view` ([#53](https://github.com/Unity-Technologies/UnityDataTools/issues/53)) |
 
 ## Related documentation
 

Documentation/command-analyze.md

@@ -238,7 +238,7 @@ When `--skip-references` is used, some functionality is lost:
 
 * the `find-refs` command will not work
 * `view_material_shader_refs` and `view_material_texture_refs` will be empty
-* `script_object_view` will be empty
+* `script_object_view` will be empty (`managed_references` is still populated)
 * `dangling_refs` will be empty
 * Queries that look at the relationship between objects will not work.  For example the refs table is required to link between a `MonoBehaviour` and its `MonoScript`.