Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions pkgs/sdk/server/src/Interfaces/DataSourceStatus.cs
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,12 @@ public struct ErrorInfo
/// </summary>
public bool FDv1Fallback { get; set; }

/// <summary>
/// When <see cref="FDv1Fallback"/> is true, how long to stay on FDv1 before attempting to
/// recover to FDv2. Null means the directive carried no TTL, so the default applies.
/// </summary>
internal TimeSpan? FDv1FallbackTtl { get; set; }

/// <summary>
/// Constructs an instance based on an exception.
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ internal sealed class CompositeSource : IDataSource, ICompositeSourceActionable
private readonly IDataSourceUpdatesV2 _originalUpdateSink;
private readonly IDataSourceUpdatesV2 _sanitizedUpdateSink;
private readonly SourcesList<(SourceFactory Factory, ActionApplierFactory ActionApplierFactory, CompositeEntryKind Kind)> _sourcesList;
private readonly List<(SourceFactory Factory, ActionApplierFactory ActionApplierFactory, CompositeEntryKind Kind)> _originalEntries;
private readonly DisableableDataSourceUpdatesTracker _disableableTracker;

// Tracks the entry from the sources list that was used to create the current
Expand Down Expand Up @@ -76,6 +77,7 @@ public CompositeSource(
// this tracker is used to disconnect the current source from the updates sink when it is no longer needed.
_disableableTracker = new DisableableDataSourceUpdatesTracker();

_originalEntries = new List<(SourceFactory Factory, ActionApplierFactory ActionApplierFactory, CompositeEntryKind Kind)>(factoryTuples);
_sourcesList = new SourcesList<(SourceFactory Factory, ActionApplierFactory ActionApplierFactory, CompositeEntryKind Kind)>(
circular: circular,
initialList: factoryTuples
Expand Down Expand Up @@ -487,6 +489,23 @@ public void BlockAll(Predicate<CompositeEntryKind> kindMatches)
});
}

public void UnblockAll()
{
if (_disposed)
{
return;
}

EnqueueAction(() =>
{
lock (_lock)
{
_sourcesList.Replace(_originalEntries);
_log.Debug("{0} restored all of its entries.", _compositeDescription);
}
});
}

private void logTransition(String previousDescription, String currentDescription) {
if (previousDescription != null && currentDescription != null) {
_log.Debug("{0} transitioned from {1} to {2}.", _compositeDescription, previousDescription, currentDescription);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,12 @@ internal interface ICompositeSourceActionable
/// </summary>
/// <param name="kindMatches">predicate selecting which kinds to remove</param>
void BlockAll(Predicate<CompositeEntryKind> kindMatches);

/// <summary>
/// Restores every entry removed by <see cref="BlockCurrent"/> or <see cref="BlockAll"/>, so
/// the source list matches its original configuration. The current data source is not
/// disposed -- callers should follow this with <see cref="GoToFirst"/>.
/// </summary>
void UnblockAll();
}
}
69 changes: 69 additions & 0 deletions pkgs/sdk/server/src/Internal/FDv2DataSources/FDv1FallbackTtl.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;

namespace LaunchDarkly.Sdk.Server.Internal.FDv2DataSources
{
/// <summary>
/// Computes the Fallback TTL of an FDv1 Fallback Directive: how long to stay off FDv2 before
/// attempting to recover to it.
/// </summary>
internal static class FDv1FallbackTtl
{
internal const string TtlHeaderName = "x-ld-fd-fallback-ttl";

/// <summary>
/// Used whenever the service supplies no usable TTL. Also the maximum valid TTL.
/// </summary>
internal static readonly TimeSpan DefaultTtl = TimeSpan.FromHours(1);

private static readonly Random Random = new Random();
private static readonly object RandomLock = new object();

/// <summary>
/// Returns the default TTL, jittered by subtracting a value drawn uniformly from
/// [0, DefaultTtl/2], so that SDKs which fell back together do not all retry at once.
/// </summary>
internal static TimeSpan Default()
{
double fraction;
lock (RandomLock)
{
fraction = Random.NextDouble();
}
return DefaultTtl - TimeSpan.FromTicks((long)(DefaultTtl.Ticks / 2 * fraction));
}

/// <summary>
/// Returns a service-supplied TTL as-is if it is within (0, 1 hour]; otherwise the
/// jittered default, so a fallback is never indefinite.
/// </summary>
internal static TimeSpan FromServiceTtl(double? seconds)
{
if (seconds.HasValue && seconds.Value > 0 && seconds.Value <= DefaultTtl.TotalSeconds)
{
return TimeSpan.FromSeconds(seconds.Value);
}
return Default();
}

/// <summary>
/// Returns the TTL carried by the X-LD-FD-Fallback-TTL header, which is a whole number
/// of seconds. A missing or malformed header gives the jittered default.
/// </summary>
internal static TimeSpan FromHeaders(IEnumerable<KeyValuePair<string, IEnumerable<string>>> headers)
{
var value = headers?
.Where(h => string.Equals(h.Key, TtlHeaderName, StringComparison.OrdinalIgnoreCase))
.SelectMany(h => h.Value)
.FirstOrDefault();
if (value != null &&
long.TryParse(value.Trim(), NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var seconds))
{
return FromServiceTtl(seconds);
}
return Default();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,15 @@ internal static class FDv2ChangeSetTranslator
/// <param name="log">Logger for diagnostic messages.</param>
/// <param name="environmentId">The environment ID to include in the changeset.</param>
/// <param name="fdv1Fallback">Whether to mark the changeset as carrying the FDv1 fallback directive.</param>
/// <param name="fdv1FallbackTtl">The directive's Fallback TTL, if any.</param>
/// <returns>A DataStoreTypes.ChangeSet containing the converted data.</returns>
/// <exception cref="ArgumentOutOfRangeException">Thrown when the changeset type is unknown.</exception>
public static DataStoreTypes.ChangeSet<DataStoreTypes.ItemDescriptor> ToChangeSet(
FDv2ChangeSet changeset,
Logger log,
string environmentId = null,
bool fdv1Fallback = false)
bool fdv1Fallback = false,
TimeSpan? fdv1FallbackTtl = null)
{
DataStoreTypes.ChangeSetType changeSetType;
switch (changeset.Type)
Expand Down Expand Up @@ -105,7 +107,8 @@ internal static class FDv2ChangeSetTranslator
changeset.Selector,
dataBuilder.ToImmutable(),
environmentId,
fdv1Fallback);
fdv1Fallback,
fdv1FallbackTtl);
}

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using System;
using System.Threading.Tasks;
using LaunchDarkly.Sdk.Server.Subsystems;

Expand All @@ -12,15 +13,18 @@ private class CompletingDataSource : IDataSource
{
private readonly IDataSource _inner;
private readonly InitializationTracker _tracker;
private readonly IDisposable _recovery;

public CompletingDataSource(IDataSource inner, InitializationTracker tracker)
public CompletingDataSource(IDataSource inner, InitializationTracker tracker, IDisposable recovery)
{
_inner = inner;
_tracker = tracker;
_recovery = recovery;
}

public void Dispose()
{
_recovery.Dispose();
_inner.Dispose();
_tracker.Dispose();
}
Expand Down
125 changes: 112 additions & 13 deletions pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,11 @@

// The FDv1 fallback applier is phase-agnostic: when the directive is observed it
// calls BlockAll(FDv2) to remove every FDv2 entry from the outer list, then GoToNext
// lands on the FDv1 fallback entry (or on exhaustion when none was configured).
// lands on the FDv1 fallback entry (when one is configured). Either way it schedules
// an FDv2 recovery attempt after the directive's Fallback TTL.
var directedFallbackRecovery = new DirectedFallbackRecovery(Any(fdv1Synchronizers), sublogger);
ActionApplierFactory fdv1FallbackApplierFactory =
(actionable) => new FDv1FallbackActionApplier(actionable);
(actionable) => new FDv1FallbackActionApplier(actionable, directedFallbackRecovery);

var initializationTracker =
new InitializationTracker(Any(initializers), Any(synchronizers), Any(fdv1Synchronizers));
Expand Down Expand Up @@ -136,7 +138,7 @@
}

return new CompletingDataSource(new CompositeSource("FDv2DataSource", updatesSink, underlyingComposites, sublogger, circular: false),
initializationTracker);
initializationTracker, directedFallbackRecovery);
}

/// <summary>
Expand Down Expand Up @@ -432,17 +434,18 @@
}

/// <summary>
/// Action applier that observes an FDv1 fallback signal and advances the outer composite
/// to the FDv1 fallback synchronizer entry. The directive may arrive either on
/// Action applier that observes an FDv1 fallback signal, advances the outer composite
/// to the FDv1 fallback synchronizer entry, and schedules FDv2 recovery after the
/// directive's Fallback TTL. The directive may arrive either on
/// <see cref="UpdateStatus"/> (errorInfo.FDv1Fallback) or on <see cref="Apply"/>
/// (changeSet.FDv1Fallback) when a successful payload also carried the directive header.
/// </summary>
/// <remarks>
/// Phase-agnostic: when attached to either the initializers entry or the synchronizers
/// entry of the outer FDv2 composite, <see cref="ICompositeSourceActionable.BlockAll"/>
/// removes every FDv2 entry in one shot, leaving the FDv1 fallback entry (if configured)
/// as the next stop. When no FDv1 fallback entry was configured the outer list is exhausted
/// and the composite halts the data system.
/// as the next stop. When no FDv1 fallback entry was configured, no source runs (the data
/// system reports Interrupted) until the recovery attempt.
///
/// The single-shot _triggered flag makes this applier idempotent in case the directive
/// arrives more than once on the same propagation chain (for example, on a successful FDv2
Expand All @@ -452,26 +455,28 @@
internal class FDv1FallbackActionApplier : IDataSourceObserver
{
private readonly ICompositeSourceActionable _actionable;
private readonly DirectedFallbackRecovery _recovery;
private int _triggered;

public FDv1FallbackActionApplier(ICompositeSourceActionable actionable)
public FDv1FallbackActionApplier(ICompositeSourceActionable actionable, DirectedFallbackRecovery recovery)
{
_actionable = actionable ?? throw new ArgumentNullException(nameof(actionable));
_recovery = recovery ?? throw new ArgumentNullException(nameof(recovery));
}

public void UpdateStatus(DataSourceState newState, DataSourceStatus.ErrorInfo? newError)
{
if (newError?.FDv1Fallback != true) return;
Trigger();
Trigger(newError.Value.FDv1FallbackTtl);
}

public void Apply(ChangeSet<ItemDescriptor> changeSet)
{
if (!changeSet.FDv1Fallback) return;
Trigger();
Trigger(changeSet.FDv1FallbackTtl);
}

private void Trigger()
private void Trigger(TimeSpan? ttl)
{
if (Interlocked.CompareExchange(ref _triggered, 1, 0) != 0) return;

Expand All @@ -480,8 +485,102 @@
// itself down by the time the directive is observed, but DisposeCurrent is
// idempotent and guards against sources that don't.
_actionable.DisposeCurrent();
_actionable.GoToNext();
_actionable.StartCurrent();
if (_recovery.HasFDv1FallbackSynchronizer)
{
_actionable.GoToNext();
_actionable.StartCurrent();
}

_recovery.Schedule(_actionable, ttl ?? FDv1FallbackTtl.Default());
}
}

/// <summary>
/// Schedules the FDv2 recovery attempt that ends a Directed Fallback once its Fallback TTL
/// elapses. Shared by every <see cref="FDv1FallbackActionApplier"/> of one FDv2 data source.
/// </summary>
/// <remarks>
/// Recovery restores the FDv2 entries and goes to the first one, re-running the full
/// initialization and synchronization flow. Going to an entry disposes and disconnects the
/// current (FDv1) source first, so FDv1 and FDv2 never write to the store at the same time.
/// A later directive replaces any pending recovery with one using the new TTL.
/// </remarks>
internal sealed class DirectedFallbackRecovery : IDisposable
{
private readonly object _lock = new object();
private readonly Logger _log;
private CancellationTokenSource _canceller;
private bool _disposed;

public bool HasFDv1FallbackSynchronizer { get; }

public DirectedFallbackRecovery(bool hasFDv1FallbackSynchronizer, Logger logger)
{
HasFDv1FallbackSynchronizer = hasFDv1FallbackSynchronizer;
_log = logger ?? throw new ArgumentNullException(nameof(logger));
}

public void Schedule(ICompositeSourceActionable actionable, TimeSpan ttl)
{
CancellationToken token;
lock (_lock)
{
if (_disposed) return;
CancelPendingRecovery();
_canceller = new CancellationTokenSource();
token = _canceller.Token;
}

if (HasFDv1FallbackSynchronizer)
{
_log.Info("Falling back to FDv1; will attempt to recover to FDv2 in {0} seconds.",
ttl.TotalSeconds);
}
else
{
_log.Warn("Received an FDv1 fallback directive, but no FDv1 fallback synchronizer is " +
"configured; will attempt to recover to FDv2 in {0} seconds.", ttl.TotalSeconds);
}

_ = Task.Run(async () =>
{
try
{
await Task.Delay(ttl, token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}

lock (_lock)
{
if (_disposed || token.IsCancellationRequested) return;
}

_log.Info("FDv1 fallback TTL elapsed; attempting to recover to FDv2.");
actionable.UnblockAll();
actionable.GoToFirst();
actionable.StartCurrent();

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (macos-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (macos-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (macos-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (macos-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (macos-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (macos-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (macos-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (macos-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (ubuntu-22.04)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (ubuntu-22.04)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (ubuntu-22.04)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (ubuntu-22.04)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (ubuntu-22.04)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (ubuntu-22.04)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (ubuntu-22.04)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (ubuntu-22.04)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (windows-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (windows-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (windows-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (windows-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (windows-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (windows-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (windows-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.

Check warning on line 564 in pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs

View workflow job for this annotation

GitHub Actions / build-and-test (windows-latest)

Because this call is not awaited, execution of the current method continues before the call is completed. Consider applying the 'await' operator to the result of the call.
});
}

public void Dispose()
{
lock (_lock)
{
_disposed = true;
CancelPendingRecovery();
}
}

// Must be called while holding _lock.
private void CancelPendingRecovery()
{
if (_canceller == null) return;
_canceller.Cancel();
_canceller.Dispose();
_canceller = null;
}
}

Expand Down
Loading
Loading