Include ref.parent matches in events by-reference navigation

src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs

@@ -108,7 +108,7 @@ public static void SetEventReference(this PersistentEvent ev, string name, strin
 
     public static string? GetSessionId(this PersistentEvent ev)
     {
-        return ev.IsSessionStart() ? ev.ReferenceId : ev.GetEventReference("session");
+        return ev.IsSessionStart() ? ev.ReferenceId : ev.GetEventReference(Event.KnownReferenceNames.Session);
     }
 
     public static void SetSessionId(this PersistentEvent ev, string sessionId)
@@ -119,7 +119,7 @@ public static void SetSessionId(this PersistentEvent ev, string sessionId)
         if (ev.IsSessionStart())
             ev.ReferenceId = sessionId;
         else
-            ev.SetEventReference("session", sessionId);
+            ev.SetEventReference(Event.KnownReferenceNames.Session, sessionId);
     }
 
     public static bool HasSessionEndTime(this PersistentEvent ev)

src/Exceptionless.Core/Extensions/StringExtensions.cs

@@ -131,7 +131,7 @@ public static bool IsNumeric(this string? value)
 
     public static bool IsValidFieldName(this string? value)
     {
-        if (value is null || value.Length > 25)
+        if (String.IsNullOrEmpty(value) || value.Length > 25)
             return false;
 
         return IsValidIdentifier(value);

src/Exceptionless.Core/Migrations/011_BackfillParentReferences.cs

@@ -0,0 +1,118 @@
+using System.Diagnostics;
+using System.Text.Json;
+using Elastic.Clients.Elasticsearch;
+using Elastic.Clients.Elasticsearch.Tasks;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Repositories.Configuration;
+using Foundatio.Repositories.Elasticsearch.Extensions;
+using Foundatio.Repositories.Migrations;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Migrations;
+
+public sealed class BackfillParentReferences : MigrationBase
+{
+    private readonly ElasticsearchClient _client;
+    private readonly ExceptionlessElasticConfiguration _config;
+    private readonly TimeProvider _timeProvider;
+
+    public BackfillParentReferences(ExceptionlessElasticConfiguration configuration, TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(loggerFactory)
+    {
+        _config = configuration;
+        _client = configuration.Client;
+        _timeProvider = timeProvider;
+
+        MigrationType = MigrationType.VersionedAndResumable;
+        Version = 11;
+    }
+
+    public override async Task RunAsync(MigrationContext context)
+    {
+        string referenceKey = $"@ref:{Event.KnownReferenceNames.Parent}";
+        string indexKey = $"{Event.KnownReferenceNames.Parent}-r";
+        // Painless trim() only handles ASCII whitespace; match CopyDataToIndex's String.Trim().
+        string whitespace = String.Concat(Enumerable.Range(Char.MinValue, Char.MaxValue + 1)
+            .Select(value => (char)value).Where(Char.IsWhiteSpace));
+        string script = $$"""
+            def parentReference = null;
+            if (ctx._source.data != null) {
+                for (def entry : ctx._source.data.entrySet()) {
+                    String key = entry.getKey();
+                    int start = 0;
+                    int end = key.length();
+                    while (start < end && params.whitespace.indexOf(key.substring(start, start + 1)) >= 0) start++;
+                    while (end > start && params.whitespace.indexOf(key.substring(end - 1, end)) >= 0) end--;
+                    if (key.substring(start, end).equalsIgnoreCase('{{referenceKey}}')) {
+                        parentReference = entry.getValue();
+                    }
+                }
+            }
+            if (parentReference != null) {
+                if (ctx._source.idx == null) ctx._source.idx = [:];
+                ctx._source.idx['{{indexKey}}'] = parentReference.toString();
+            } else {
+                ctx.op = 'noop';
+            }
+            """;
+
+        _logger.LogInformation("Backfilling retained event parent references");
+        var stopwatch = Stopwatch.StartNew();
+        var response = await _client.UpdateByQueryAsync<PersistentEvent>(request => request
+            .Indices($"{_config.Events.VersionedName}-*")
+            .Query(query => query.Bool(filter => filter.MustNot(mustNot => mustNot.Exists(exists => exists.Field($"idx.{indexKey}")))))
+            .Script(value => value.Source(script).Lang(ScriptLanguage.Painless)
+                .Params(new Dictionary<string, object> { ["whitespace"] = whitespace }))
+            .WaitForCompletion(false), context.CancellationToken);
+        _logger.LogRequest(response, LogLevel.Information);
+
+        if (!response.IsValidResponse || response.Task is null)
+            throw new ApplicationException($"Unable to start parent-reference backfill: {response.DebugInformation}");
+
+        int attempts = 0;
+        while (!context.CancellationToken.IsCancellationRequested)
+        {
+            var taskStatus = await _client.Tasks.GetAsync(response.Task.FullyQualifiedId, context.CancellationToken);
+            if (!taskStatus.IsValidResponse)
+                throw new ApplicationException($"Unable to monitor parent-reference backfill: {taskStatus.DebugInformation}");
+
+            if (taskStatus.Completed)
+            {
+                EnsureTaskSucceeded(taskStatus);
+                _logger.LogInformation("Finished parent-reference backfill: Duration={Duration}", stopwatch.Elapsed);
+                return;
+            }
+
+            attempts++;
+            await context.Lock.RenewAsync();
+            await Task.Delay(TimeSpan.FromSeconds(attempts <= 5 ? 1 : 5), _timeProvider, context.CancellationToken);
+        }
+
+        context.CancellationToken.ThrowIfCancellationRequested();
+    }
+
+    internal static void EnsureTaskSucceeded(GetTasksResponse taskStatus)
+    {
+        if (taskStatus.Error is not null)
+            throw new ApplicationException($"Parent-reference backfill failed: {JsonSerializer.Serialize(taskStatus.Error)}");
+
+        if (taskStatus.Response is null)
+            return;
+
+        JsonElement response = JsonSerializer.SerializeToElement(taskStatus.Response);
+        if (response.ValueKind == JsonValueKind.Object
+            && response.TryGetProperty("version_conflicts", out var versionConflicts)
+            && versionConflicts.TryGetInt64(out long conflictCount)
+            && conflictCount > 0)
+        {
+            throw new ApplicationException($"Parent-reference backfill failed with {conflictCount} version conflicts.");
+        }
+
+        if (response.ValueKind == JsonValueKind.Object
+            && response.TryGetProperty("failures", out var failures)
+            && failures.ValueKind == JsonValueKind.Array
+            && failures.GetArrayLength() > 0)
+        {
+            throw new ApplicationException($"Parent-reference backfill failed: {failures}");
+        }
+    }
+}

src/Exceptionless.Core/Models/Event.cs

@@ -187,6 +187,12 @@ public static class KnownTags
         public const string Internal = "Internal";
     }
 
+    public static class KnownReferenceNames
+    {
+        public const string Parent = "parent";
+        public const string Session = "session";
+    }
+
     public static class KnownDataKeys
     {
         public const string Error = "@error";

src/Exceptionless.Core/Pipeline/035_CopySimpleDataToIdxAction.cs

@@ -1,4 +1,5 @@
-using Exceptionless.Core.Plugins.EventProcessor;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Plugins.EventProcessor;
 using Microsoft.Extensions.Logging;
 
 namespace Exceptionless.Core.Pipeline;
@@ -11,7 +12,15 @@ public CopySimpleDataToIdxAction(AppOptions options, ILoggerFactory loggerFactor
     public override Task ProcessAsync(EventContext ctx)
     {
         if (!ctx.Organization.HasPremiumFeatures)
+        {
+            string[] parentKeys = ctx.Event.Data?.Keys
+                .Where(key => String.Equals(key.Trim(), $"@ref:{Event.KnownReferenceNames.Parent}", StringComparison.OrdinalIgnoreCase))
+                .ToArray() ?? [];
+            if (parentKeys.Length > 0)
+                ctx.Event.CopyDataToIndex(parentKeys);
+
             return Task.CompletedTask;
+        }
 
         // TODO: Do we need a pipeline action to trim keys and remove null values that may be sent by other native clients.
         ctx.Event.CopyDataToIndex([]);

src/Exceptionless.Core/Repositories/Queries/Validation/PersistentEventQueryValidator.cs

@@ -1,4 +1,5 @@
-using Exceptionless.Core.Repositories.Configuration;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Repositories.Configuration;
 using Foundatio.Parsers.LuceneQueries;
 using Foundatio.Parsers.LuceneQueries.Visitors;
 using Microsoft.Extensions.Logging;
@@ -12,6 +13,7 @@ public sealed class PersistentEventQueryValidator : AppQueryValidator
             "date",
             "type",
             EventIndex.Alias.ReferenceId,
+            $"idx.{Event.KnownReferenceNames.Parent}-r",
             "reference_id",
             EventIndex.Alias.OrganizationId,
             "organization_id",
@@ -101,9 +103,11 @@ internal static bool IsFreeQueryField(string field)
 
     protected override QueryProcessResult ApplyQueryRules(QueryValidationResult result)
     {
+        bool hasInvalidReferenceField = result.ReferencedFields.Any(field => field.StartsWith("ref.", StringComparison.OrdinalIgnoreCase));
         return new QueryProcessResult
         {
-            IsValid = result.IsValid,
+            IsValid = result.IsValid && !hasInvalidReferenceField,
+            Message = hasInvalidReferenceField ? "Invalid reference field name" : null,
             UsesPremiumFeatures = !result.ReferencedFields.All(IsFreeQueryField)
         };
     }

src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs

@@ -73,7 +73,7 @@ public override Task VisitAsync(MissingNode node, IQueryVisitorContext context)
             return null;
 
         string[] parts = field.Split('.');
-        if (parts.Length != 2 || (parts.Length == 2 && parts[1].StartsWith("@")))
+        if (parts.Length != 2 || parts[1].StartsWith("@") || !parts[1].IsValidFieldName())
             return field;
 
         if (String.Equals(parts[0], "data", StringComparison.OrdinalIgnoreCase))

src/Exceptionless.Web/Api/Handlers/EventHandler.cs

@@ -16,19 +16,19 @@
 using Exceptionless.Core.Validation;
 using Exceptionless.DateTimeExtensions;
 using Exceptionless.Web.Api.Infrastructure;
-using Exceptionless.Web.Api.Results;
 using Exceptionless.Web.Api.Messages;
+using Exceptionless.Web.Api.Results;
 using Exceptionless.Web.Extensions;
 using Exceptionless.Web.Models;
 using Exceptionless.Web.Utility;
 using Foundatio.Caching;
 using Foundatio.Mediator;
 using Foundatio.Queues;
-using Foundatio.Serializer;
 using Foundatio.Repositories;
 using Foundatio.Repositories.Elasticsearch.Extensions;
 using Foundatio.Repositories.Extensions;
 using Foundatio.Repositories.Models;
+using Foundatio.Serializer;
 using Microsoft.Net.Http.Headers;
 
 namespace Exceptionless.Web.Api.Handlers;
@@ -232,7 +232,7 @@ public async Task<Result<PagedResult<object>>> Handle(GetEventsByReferenceId mes
 
         var ti = TimeRangeParser.GetTimeInfo(null, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider));
         var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true };
-        return await GetInternalAsync(sf, ti, httpContext, String.Concat("reference:", message.ReferenceId), null, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include));
+        return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.ReferenceId} OR ref.{Event.KnownReferenceNames.Parent}:{message.ReferenceId})", null, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include));
     }
 
     public async Task<Result<PagedResult<object>>> Handle(GetEventsByReferenceIdAndProject message)
@@ -251,7 +251,7 @@ public async Task<Result<PagedResult<object>>> Handle(GetEventsByReferenceIdAndP
 
         var ti = TimeRangeParser.GetTimeInfo(null, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, appOptions.MaximumRetentionDays, timeProvider));
         var sf = new AppFilter(project, organization);
-        return await GetInternalAsync(sf, ti, httpContext, String.Concat("reference:", message.ReferenceId), null, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include));
+        return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.ReferenceId} OR ref.{Event.KnownReferenceNames.Parent}:{message.ReferenceId})", null, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include));
     }
 
     public async Task<Result<PagedResult<object>>> Handle(GetEventsBySessionId message)

src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts

@@ -388,7 +388,7 @@ export function getEventQuery(request: GetEventRequest) {
 }
 
 export function getEventsByReferenceQuery(request: GetEventsByReferenceRequest) {
-    return createQuery<EventSummaryModel<SummaryTemplateKeys>[], ProblemDetails>(() => ({
+    return createQuery<FetchClientResponse<EventSummaryModel<SummaryTemplateKeys>[]>, ProblemDetails>(() => ({
         enabled: () => !!accessToken.current && !!request.route.referenceId,
         queryFn: async () => {
             const client = useFetchClient();
@@ -405,11 +405,12 @@ export function getEventsByReferenceQuery(request: GetEventsByReferenceRequest)
                     limit: 20,
                     mode: 'summary',
                     page: 1,
-                    ...request.params
+                    ...request.params,
+                    include: 'total'
                 }
             });
 
-            return response.data!;
+            return response;
         },
         queryKey: queryKeys.eventsByReference(request.route.referenceId, request.route.projectId, request.params)
     }));

src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.test.ts

@@ -86,6 +86,16 @@ describe('TagFilter', () => {
     });
 });
 
+describe('ReferenceFilter', () => {
+    it('matches direct and parent references', () => {
+        expect(new ReferenceFilter('ref-123').toFilter()).toBe('(reference:"ref-123" OR ref.parent:"ref-123")');
+    });
+
+    it('quotes the reference in both clauses', () => {
+        expect(new ReferenceFilter('ref 123').toFilter()).toBe('(reference:"ref 123" OR ref.parent:"ref 123")');
+    });
+});
+
 describe('applyTimeFilter', () => {
     it('removes an existing date filter when time is explicitly empty', () => {
         // Arrange

src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/models.svelte.ts

@@ -273,7 +273,8 @@ export class ReferenceFilter implements IFilter {
             return '';
         }
 
-        return `reference:${quoteIfSpecialCharacters(this.value)}`;
+        const reference = quoteIfSpecialCharacters(this.value);
+        return `(reference:${reference} OR ref.parent:${reference})`;
     }
 }
 

src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte

@@ -47,16 +47,20 @@
     let references = $derived.by(() => {
         let refs: { id: string; name: string }[] = [];
         Object.entries(event.data || {}).forEach(([key, value]) => {
-            if (key.startsWith(referencePrefix)) {
+            if (key.startsWith(referencePrefix) && value != null) {
                 refs.push({
-                    id: value as string,
+                    id: String(value),
                     name: key.slice(5)
                 });
             }
         });
         return refs;
     });
 
+    function isValidReferenceName(name: string): boolean {
+        return !/[\uD800-\uDFFF]/.test(name) && /^[\p{L}\p{Nd}-]{1,25}$/u.test(name);
+    }
+
     let level = $derived(event.data?.['@level']?.toLowerCase());
     let location = $derived(getLocation(event));
 
@@ -110,12 +114,20 @@
         {/if}
         {#each references as reference (reference.id)}
             <Table.Row class="group">
-                {#if reference.name === 'session'}
+                {#if reference.name.toLowerCase() === 'session'}
                     <Table.Head class="w-40 font-semibold whitespace-nowrap">Session</Table.Head>
                     <Table.Cell class="w-4 pr-0"><EventsFacetedFilter.SessionTrigger changed={filterChanged} value={reference.id} /></Table.Cell>
-                {:else}
+                {:else if reference.name.toLowerCase() === 'parent'}
                     <Table.Head class="w-40 font-semibold whitespace-nowrap">{reference.name}</Table.Head>
                     <Table.Cell class="w-4 pr-0"><EventsFacetedFilter.ReferenceTrigger changed={filterChanged} value={reference.id} /></Table.Cell>
+                {:else if isValidReferenceName(reference.name)}
+                    <Table.Head class="w-40 font-semibold whitespace-nowrap">{reference.name}</Table.Head>
+                    <Table.Cell class="w-4 pr-0"
+                        ><EventsFacetedFilter.StringTrigger changed={filterChanged} term={`ref.${reference.name}`} value={reference.id} /></Table.Cell
+                    >
+                {:else}
+                    <Table.Head class="w-40 font-semibold whitespace-nowrap">{reference.name}</Table.Head>
+                    <Table.Cell class="w-4 pr-0"></Table.Cell>
                 {/if}
                 <Table.Cell
                     ><A