From e97ca84403adbd1d42976359e443765f8791058e Mon Sep 17 00:00:00 2001 From: Florian Rappl Date: Sat, 1 Aug 2026 10:40:44 +0200 Subject: [PATCH 01/10] Remove badges --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 9bb7dd8..0ac9e96 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,6 @@ [![GitHub Tag](https://img.shields.io/github/tag/AngleSharp/AngleSharp.Io.svg?style=flat-square)](https://github.com/AngleSharp/AngleSharp.Io/releases) [![NuGet Count](https://img.shields.io/nuget/dt/AngleSharp.Io.svg?style=flat-square)](https://www.nuget.org/packages/AngleSharp.Io/) [![Issues Open](https://img.shields.io/github/issues/AngleSharp/AngleSharp.Io.svg?style=flat-square)](https://github.com/AngleSharp/AngleSharp.Io/issues) -[![Gitter Chat](http://img.shields.io/badge/gitter-AngleSharp/AngleSharp-blue.svg?style=flat-square)](https://gitter.im/AngleSharp/AngleSharp) -[![StackOverflow Questions](https://img.shields.io/stackexchange/stackoverflow/t/anglesharp.svg?style=flat-square)](https://stackoverflow.com/tags/anglesharp) [![CLA Assistant](https://cla-assistant.io/readme/badge/AngleSharp/AngleSharp.Io?style=flat-square)](https://cla-assistant.io/AngleSharp/AngleSharp.Io) AngleSharp.Io extends AngleSharp with powerful requesters, caching mechanisms, and storage systems. It is coupled more strongly to the underlying operating system than AngleSharp itself. Therefore it has stronger dependencies and demands and cannot be released for the standard framework (4.6). Nevertheless, it is released as a .NET Standard 2.0 library. From 9165d812067e66999298c00730ca3e8d276b20ea Mon Sep 17 00:00:00 2001 From: Marko Lahma Date: Sat, 8 Aug 2026 15:31:07 +0300 Subject: [PATCH 02/10] Move the build orchestrator to the Fallout stable channel (10.4.0) Fallout v10.4.0 is the first stable-channel release; the previously pinned 11.0.18 belongs to the edge channel. The stable CLI ships as Fallout.GlobalTool (the fallout command is unchanged), and Fallout.Common 10.4.0 resolves the patched System.Security.Cryptography.Xml 10.0.10 on its own, so the manual transitive pin is no longer needed (Fallout-build/Fallout#618). Co-Authored-By: Claude Fable 5 --- .config/dotnet-tools.json | 4 ++-- build/_build.csproj | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 6a8c8a1..a8483d3 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -2,8 +2,8 @@ "version": 1, "isRoot": true, "tools": { - "fallout.cli": { - "version": "11.0.18", + "fallout.globaltool": { + "version": "10.4.0", "commands": [ "fallout" ] diff --git a/build/_build.csproj b/build/_build.csproj index 647ac26..11c89b0 100644 --- a/build/_build.csproj +++ b/build/_build.csproj @@ -11,9 +11,7 @@ - - - + From 47ea8fd2f1f56977a78d3050491428c05c70b01c Mon Sep 17 00:00:00 2001 From: Florian Rappl Date: Thu, 3 Sep 2026 12:40:45 +0200 Subject: [PATCH 03/10] Migrated to new test site --- .../Cookie/ClassicTests.cs | 16 ++++++------ .../Network/CookieHandlingTests.cs | 25 ++++++++++--------- .../Network/HttpClientRequesterTests.cs | 2 +- .../Network/WebSocketTests.cs | 15 ++++------- 4 files changed, 26 insertions(+), 32 deletions(-) diff --git a/src/AngleSharp.Io.Tests/Cookie/ClassicTests.cs b/src/AngleSharp.Io.Tests/Cookie/ClassicTests.cs index c52a7db..b23b200 100644 --- a/src/AngleSharp.Io.Tests/Cookie/ClassicTests.cs +++ b/src/AngleSharp.Io.Tests/Cookie/ClassicTests.cs @@ -196,7 +196,7 @@ public async Task SettingOneCookiesInOneRequestAppearsInDocument() { if (Helper.IsNetworkAvailable()) { - var url = "https://httpbingo.org/cookies/set?k1=v1"; + var url = "https://anglesharp-tests.anglevisions.com/test-cases/set-cookies?k1=v1"; var config = Configuration.Default.WithCookies().WithDefaultLoader(); var context = BrowsingContext.New(config); var document = await context.OpenAsync(url); @@ -210,7 +210,7 @@ public async Task SettingTwoCookiesInOneRequestAppearsInDocument() { if (Helper.IsNetworkAvailable()) { - var url = "https://httpbingo.org/cookies/set?k2=v2&k1=v1"; + var url = "https://anglesharp-tests.anglevisions.com/test-cases/set-cookies?k2=v2&k1=v1"; var config = Configuration.Default.WithCookies().WithDefaultLoader(); var context = BrowsingContext.New(config); var document = await context.OpenAsync(url); @@ -225,7 +225,7 @@ public async Task SettingThreeCookiesInOneRequestAppearsInDocument() { if (Helper.IsNetworkAvailable()) { - var url = "https://httpbingo.org/cookies/set?test=baz&k2=v2&k1=v1&foo=bar"; + var url = "https://anglesharp-tests.anglevisions.com/test-cases/set-cookies?test=baz&k2=v2&k1=v1&foo=bar"; var config = Configuration.Default.WithCookies().WithDefaultLoader(); var context = BrowsingContext.New(config); var document = await context.OpenAsync(url); @@ -240,12 +240,10 @@ public async Task SettingThreeCookiesInOneRequestAreTransportedToNextRequest() { if (Helper.IsNetworkAvailable()) { - var baseUrl = "https://httpbingo.org/cookies"; - var url = baseUrl + "/set?test=baz&k2=v2&k1=v1&foo=bar"; var config = Configuration.Default.WithCookies().WithDefaultLoader(); var context = BrowsingContext.New(config); - await context.OpenAsync(url); - var document = await context.OpenAsync(baseUrl); + await context.OpenAsync("https://anglesharp-tests.anglevisions.com/test-cases/set-cookies?test=baz&k2=v2&k1=v1&foo=bar"); + var document = await context.OpenAsync("https://anglesharp-tests.anglevisions.com/test-cases/get-cookies"); AssertCookies(document.Body.TextContent, new KeyValuePair("foo", "bar"), @@ -260,8 +258,8 @@ public async Task SettingCookieIsPreservedViaRedirect() { if (Helper.IsNetworkAvailable()) { - var cookieUrl = "https://httpbingo.org/cookies/set?test=baz"; - var redirectUrl = "https://httpbingo.org/redirect-to?url=https%3A%2F%2Fhttpbingo.org%2Fcookies"; + var cookieUrl = "https://anglesharp-tests.anglevisions.com/test-cases/set-cookies?test=baz"; + var redirectUrl = "https://anglesharp-tests.anglevisions.com/test-cases/redirect?url=https%3A%2F%2Fanglesharp-tests.anglevisions.com%2Ftest-cases%2Fget-cookies"; var config = Configuration.Default.WithCookies().WithDefaultLoader(); var context = BrowsingContext.New(config); await context.OpenAsync(cookieUrl); diff --git a/src/AngleSharp.Io.Tests/Network/CookieHandlingTests.cs b/src/AngleSharp.Io.Tests/Network/CookieHandlingTests.cs index 663e4a1..65d52fa 100644 --- a/src/AngleSharp.Io.Tests/Network/CookieHandlingTests.cs +++ b/src/AngleSharp.Io.Tests/Network/CookieHandlingTests.cs @@ -16,7 +16,7 @@ public async Task SettingOneCookiesInOneRequestAppearsInDocument() { if (Helper.IsNetworkAvailable()) { - var url = "https://httpbingo.org/cookies/set?k1=v1"; + var url = "https://anglesharp-tests.anglevisions.com/test-cases/set-cookies?k1=v1"; var config = Configuration.Default.WithCookies().WithDefaultLoader(); var context = BrowsingContext.New(config); var document = await context.OpenAsync(url); @@ -30,7 +30,7 @@ public async Task SettingTwoCookiesInOneRequestAppearsInDocument() { if (Helper.IsNetworkAvailable()) { - var url = "https://httpbingo.org/cookies/set?k2=v2&k1=v1"; + var url = "https://anglesharp-tests.anglevisions.com/test-cases/set-cookies?k2=v2&k1=v1"; var config = Configuration.Default.WithCookies().WithDefaultLoader(); var context = BrowsingContext.New(config); var document = await context.OpenAsync(url); @@ -44,7 +44,7 @@ public async Task SettingThreeCookiesInOneRequestAppearsInDocument() { if (Helper.IsNetworkAvailable()) { - var url = "https://httpbingo.org/cookies/set?test=baz&k2=v2&k1=v1&foo=bar"; + var url = "https://anglesharp-tests.anglevisions.com/test-cases/set-cookies?test=baz&k2=v2&k1=v1&foo=bar"; var config = Configuration.Default.WithCookies().WithDefaultLoader(); var context = BrowsingContext.New(config); var document = await context.OpenAsync(url); @@ -58,12 +58,12 @@ public async Task SettingThreeCookiesInOneRequestAreTransportedToNextRequest() { if (Helper.IsNetworkAvailable()) { - var baseUrl = "https://httpbingo.org/cookies"; - var url = baseUrl + "/set?test=baz&k2=v2&k1=v1&foo=bar"; + var baseUrl = "https://anglesharp-tests.anglevisions.com/test-cases"; + var url = baseUrl + "/set-cookies?test=baz&k2=v2&k1=v1&foo=bar"; var config = Configuration.Default.WithCookies().WithDefaultLoader(); var context = BrowsingContext.New(config); await context.OpenAsync(url); - var document = await context.OpenAsync(baseUrl); + var document = await context.OpenAsync(baseUrl + "/get-cookies"); AssertCookies(document.Body.TextContent, new KeyValuePair("foo", "bar"), @@ -78,9 +78,9 @@ public async Task SettingCookieIsPreservedViaRedirect() { if (Helper.IsNetworkAvailable()) { - var cookieUrl = "https://httpbingo.org/cookies/set?test=baz"; - var redirectUrl = "https://httpbingo.org/redirect-to?url=https%3A%2F%2Fhttpbingo.org%2Fcookies"; - var config = Configuration.Default.WithCookies().WithDefaultLoader(); + var cookieUrl = "https://anglesharp-tests.anglevisions.com/test-cases/set-cookies?test=baz"; + var redirectUrl = "https://anglesharp-tests.anglevisions.com/test-cases/redirect?url=https%3A%2F%2Fanglesharp-tests.anglevisions.com%2Ftest-cases%2Fget-cookies"; + var config = Configuration.Default.WithCookies().WithDefaultLoader(); var context = BrowsingContext.New(config); await context.OpenAsync(cookieUrl); var document = await context.OpenAsync(redirectUrl); @@ -95,14 +95,15 @@ public async Task SettingCookieIsPreservedViaRedirectToDifferentProtocol() { if (Helper.IsNetworkAvailable()) { - var cookieUrl = "https://httpbingo.org/cookies/set?test=baz"; - var redirectUrl = "http://httpbingo.org/redirect-to?url=http%3A%2F%2Fhttpbingo.org%2Fcookies"; + var cookieUrl = "https://anglesharp-tests.anglevisions.com/test-cases/set-cookies?test=baz"; + var redirectUrl = "http://anglesharp-tests.anglevisions.com/test-cases/redirect?url=http%3A%2F%2Fanglesharp-tests.anglevisions.com%2Ftest-cases%2Fget-cookies"; var config = Configuration.Default.WithCookies().WithDefaultLoader(); var context = BrowsingContext.New(config); await context.OpenAsync(cookieUrl); var document = await context.OpenAsync(redirectUrl); - AssertCookies(document.Body.TextContent); + AssertCookies(document.Body.TextContent, + new KeyValuePair("test", "baz")); } } diff --git a/src/AngleSharp.Io.Tests/Network/HttpClientRequesterTests.cs b/src/AngleSharp.Io.Tests/Network/HttpClientRequesterTests.cs index 8244f46..6070df8 100644 --- a/src/AngleSharp.Io.Tests/Network/HttpClientRequesterTests.cs +++ b/src/AngleSharp.Io.Tests/Network/HttpClientRequesterTests.cs @@ -131,7 +131,7 @@ public async Task EndToEnd() var requester = new HttpClientRequester(httpClient); var configuration = Configuration.Default.With(requester).WithDefaultLoader(); var context = BrowsingContext.New(configuration); - var request = DocumentRequest.Get(Url.Create("https://httpbingo.org/html")); + var request = DocumentRequest.Get(Url.Create("https://anglesharp-tests.anglevisions.com/test-cases/html")); // ACT var response = await context.GetService().FetchAsync(request).Task; diff --git a/src/AngleSharp.Io.Tests/Network/WebSocketTests.cs b/src/AngleSharp.Io.Tests/Network/WebSocketTests.cs index b710107..5c62663 100644 --- a/src/AngleSharp.Io.Tests/Network/WebSocketTests.cs +++ b/src/AngleSharp.Io.Tests/Network/WebSocketTests.cs @@ -21,8 +21,8 @@ public async Task ConnectToWebSocketEcho() var haserror = false; var messages = new List(); var closed = new TaskCompletionSource(); - var document = await BrowsingContext.New().OpenNewAsync("https://echo.websocket.org/"); - var ws = new WebSocket(document.DefaultView, "wss://echo.websocket.org/"); + var document = await BrowsingContext.New().OpenNewAsync("https://anglesharp-tests.anglevisions.com/"); + var ws = new WebSocket(document.DefaultView, "wss://anglesharp-tests.anglevisions.com/ws-echo"); // ACT ws.Opened += (s, ev) => ws.Send(message); @@ -30,11 +30,7 @@ public async Task ConnectToWebSocketEcho() { var msg = ev as MessageEvent; messages.Add(msg.Data.ToString()); - - if (messages.Count == 2) - { - ws.Close(); - } + ws.Close(); }; ws.Closed += (s, ev) => { @@ -49,9 +45,8 @@ public async Task ConnectToWebSocketEcho() // ASSERT haserror.Should().BeFalse(); - messages.Count.Should().Be(2); - messages[0].Should().StartWith("Request served by "); - messages[1].Should().Be(message); + messages.Count.Should().Be(1); + messages[0].Should().Be(message); } } } From 444772ac4da9e5b9cb5a365a50f0b71a2cac2f76 Mon Sep 17 00:00:00 2001 From: Florian Rappl Date: Mon, 19 Oct 2026 17:54:12 +0200 Subject: [PATCH 04/10] Add an HTTP cache for the requesters WithHttpCache wraps the registered HTTP(S) requesters in a CachingRequester backed by a private RFC 9111 cache. Entries are kept per origin through IHttpCacheHandler, with memory and persisted implementations mirroring the storage handlers. IoOptions gains CreateTemporary/CreatePersistent to put cookies, storage and the HTTP cache under one root directory. Claude-Session: https://claude.ai/code/session_01Je6aX4Be36K1jC6MTpDng4 --- AGENTS.md | 3 + CHANGELOG.md | 7 + README.md | 26 + .../Http/HttpCacheTests.cs | 638 ++++++++++++++++++ src/AngleSharp.Io/Http/CacheControl.cs | 147 ++++ src/AngleSharp.Io/Http/CachingRequester.cs | 69 ++ src/AngleSharp.Io/Http/HttpCache.cs | 386 +++++++++++ src/AngleSharp.Io/Http/HttpCacheEntry.cs | 57 ++ src/AngleSharp.Io/Http/HttpCacheMode.cs | 37 + src/AngleSharp.Io/Http/HttpCacheOptions.cs | 40 ++ src/AngleSharp.Io/Http/HttpCachePolicy.cs | 317 +++++++++ src/AngleSharp.Io/Http/IHttpCache.cs | 60 ++ src/AngleSharp.Io/Http/IHttpCacheHandler.cs | 48 ++ .../Http/MemoryHttpCacheHandler.cs | 93 +++ .../Http/PersistenceHttpCacheHandler.cs | 266 ++++++++ src/AngleSharp.Io/Http/PrefixedStream.cs | 86 +++ .../IoConfigurationExtensions.cs | 81 +++ src/AngleSharp.Io/IoOptions.cs | 46 ++ 18 files changed, 2407 insertions(+) create mode 100644 src/AngleSharp.Io.Tests/Http/HttpCacheTests.cs create mode 100644 src/AngleSharp.Io/Http/CacheControl.cs create mode 100644 src/AngleSharp.Io/Http/CachingRequester.cs create mode 100644 src/AngleSharp.Io/Http/HttpCache.cs create mode 100644 src/AngleSharp.Io/Http/HttpCacheEntry.cs create mode 100644 src/AngleSharp.Io/Http/HttpCacheMode.cs create mode 100644 src/AngleSharp.Io/Http/HttpCacheOptions.cs create mode 100644 src/AngleSharp.Io/Http/HttpCachePolicy.cs create mode 100644 src/AngleSharp.Io/Http/IHttpCache.cs create mode 100644 src/AngleSharp.Io/Http/IHttpCacheHandler.cs create mode 100644 src/AngleSharp.Io/Http/MemoryHttpCacheHandler.cs create mode 100644 src/AngleSharp.Io/Http/PersistenceHttpCacheHandler.cs create mode 100644 src/AngleSharp.Io/Http/PrefixedStream.cs diff --git a/AGENTS.md b/AGENTS.md index 1311aec..7bbd2be 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,9 @@ The main integration surface is `src/AngleSharp.Io/IoConfigurationExtensions.cs` - `WithCookies(...)` wires `AdvancedCookieProvider` (memory or file-backed) as the cookie service. - `WithDownload(...)` / `WithStandardDownload(...)` wraps AngleSharp's document factory to intercept attachments / binary responses. +- `WithHttpCache(...)` wraps the registered HTTP(S) requesters in `CachingRequester` + (`src/AngleSharp.Io/Http`), so it must be called after `WithRequesters(...)`. Not to be + confused with `Cache/` (`window.caches`, the DOM Cache Storage API). Cookie subsystem layout (`src/AngleSharp.Io/Cookie`): diff --git a/CHANGELOG.md b/CHANGELOG.md index 53d59d8..59aa91f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# 1.2.0 + +Released on tbd. + +- Added an HTTP cache for the requesters (`WithHttpCache`) +- Added `IoOptions.CreateTemporary` and `IoOptions.CreatePersistent` + # 1.1.0 Released on Friday, July 31 2026. diff --git a/README.md b/README.md index 0ac9e96..045ebe4 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,11 @@ var config = Configuration.Default `WithIo` wires navigator, cookies, requesters, storage, IndexedDB, and cache in one call. Clipboard and geolocation are only enabled when platforms are provided. +To keep all browsing data in one place, start from `IoOptions.CreateTemporary()` (memory only) or +`IoOptions.CreatePersistent(rootDirectory)`, which stores the cookies (`cookies.txt`), the local +storage (`storage`), and the HTTP cache (`http-cache`) below the given directory. Both enable the +HTTP cache; a plain `new IoOptions()` leaves it off. + ### Requesters If you just want to use *all* available requesters provided by AngleSharp.Io you can do the following: @@ -75,6 +80,26 @@ var config = Configuration.Default .WithDefaultLoader(); ``` +### HTTP Cache + +The HTTP(S) requesters can be wrapped with a private HTTP cache following RFC 9111. Only GET +responses are stored (HEAD is answered from them), unsafe methods such as POST invalidate the +stored responses, and `Cache-Control`, `Expires`, `Vary`, `ETag`, and `Last-Modified` are respected. +Stored responses never contain `Set-Cookie`, so cookies are not applied twice. Entries are kept +per origin of the requested address. + +```cs +var config = Configuration.Default + .WithRequesters() + .WithHttpCache() // in memory; must come after the requesters + .WithDefaultLoader(); +``` + +To persist the cache, pass a directory: `.WithHttpCache(cacheDirectory)`. More options are +available via `HttpCacheOptions`, e.g., `Mode = HttpCacheMode.OnlyIfCached` to replay previously +cached pages without network access. The cache is registered as `IHttpCache`, which can be used +to inspect or clear the stored responses. + ### Cookies To get improved cookie support, e.g., do @@ -433,6 +458,7 @@ The `SaveToAsync` (as well as the `CopyToAsync`) are extension methods for the ` - WebSockets (mostly interesting for scripting engines, e.g., JS) - Storage support with a unified `Storage` implementation - Improved cookie container (`AdvancedCookieContainer`) +- Private HTTP cache for the requesters (memory or persisted) - Enhanced download capabilities for resources / links - Web Locks support for origin-scoped request serialization - Clipboard support with injectable platform implementation diff --git a/src/AngleSharp.Io.Tests/Http/HttpCacheTests.cs b/src/AngleSharp.Io.Tests/Http/HttpCacheTests.cs new file mode 100644 index 0000000..dc74a2e --- /dev/null +++ b/src/AngleSharp.Io.Tests/Http/HttpCacheTests.cs @@ -0,0 +1,638 @@ +namespace AngleSharp.Io.Tests.Http +{ + using AngleSharp; + using AngleSharp.Dom; + using AngleSharp.Io; + using AngleSharp.Io.Http; + using AngleSharp.Io.Network; + using AngleSharp.Io.Tests.Mocks; + using NUnit.Framework; + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using HttpMethod = AngleSharp.Io.HttpMethod; + + [TestFixture] + public class HttpCacheTests + { + private static readonly DateTimeOffset Start = new DateTimeOffset(2026, 10, 19, 12, 0, 0, TimeSpan.Zero); + + [Test] + public async Task FreshResponseIsServedFromCache() + { + var server = new Server(_ => Ok("hello", ("Cache-Control", "max-age=60"))); + var cache = CreateCache(out var clock); + + var first = await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + clock.Advance(TimeSpan.FromSeconds(30)); + var second = await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(1, server.Requests.Count); + Assert.AreEqual("hello", Read(first)); + Assert.AreEqual("hello", Read(second)); + Assert.AreEqual("30", second.Headers[HeaderNames.Age]); + } + + [Test] + public async Task FragmentIsIgnoredForTheKey() + { + var server = new Server(_ => Ok("hello", ("Cache-Control", "max-age=60"))); + var cache = CreateCache(out _); + + await cache.SendAsync(Get("https://a.example/page#one"), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://a.example/page#two"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(1, server.Requests.Count); + } + + [Test] + public async Task ExpiredResponseWithoutValidatorIsFetchedAgain() + { + var server = new Server(_ => Ok("hello", ("Cache-Control", "max-age=10"))); + var cache = CreateCache(out var clock); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + clock.Advance(TimeSpan.FromSeconds(11)); + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(2, server.Requests.Count); + Assert.IsFalse(server.Requests[1].Headers.ContainsKey(HeaderNames.IfNoneMatch)); + } + + [Test] + public async Task PostIsNeitherCachedNorServedFromCache() + { + var server = new Server(_ => Ok("hello", ("Cache-Control", "max-age=60"))); + var cache = CreateCache(out _); + + await cache.SendAsync(Create(HttpMethod.Post, "https://a.example/form"), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Create(HttpMethod.Post, "https://a.example/form"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(2, server.Requests.Count); + Assert.IsNull(await cache.MatchAsync(Get("https://a.example/form"), CancellationToken.None)); + } + + [Test] + public async Task SuccessfulPostInvalidatesTargetAndLocation() + { + var server = new Server(request => request.Method == HttpMethod.Post ? + Ok("created", ("Location", "/items/1")) : + Ok("item", ("Cache-Control", "max-age=60"))); + var cache = CreateCache(out _); + + await cache.SendAsync(Get("https://a.example/items"), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://a.example/items/1"), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Create(HttpMethod.Post, "https://a.example/items"), server.SendAsync, CancellationToken.None); + + Assert.IsNull(await cache.MatchAsync(Get("https://a.example/items"), CancellationToken.None)); + Assert.IsNull(await cache.MatchAsync(Get("https://a.example/items/1"), CancellationToken.None)); + } + + [Test] + public async Task NoStoreResponseIsNotStored() + { + var server = new Server(_ => Ok("secret", ("Cache-Control", "no-store, max-age=60"))); + var cache = CreateCache(out _); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(2, server.Requests.Count); + } + + [Test] + public async Task PrivateResponseIsStoredByPrivateCache() + { + var server = new Server(_ => Ok("mine", ("Cache-Control", "private, max-age=60"))); + var cache = CreateCache(out _); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(1, server.Requests.Count); + } + + [Test] + public async Task AuthorizedRequestIsOnlyStoredWithPublicResponse() + { + var server = new Server(_ => Ok("data", ("Cache-Control", "max-age=60"))); + var cache = CreateCache(out _); + + await cache.SendAsync(Get("https://a.example/a", ("Authorization", "Bearer x")), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://a.example/a", ("Authorization", "Bearer x")), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(2, server.Requests.Count); + + server.Respond = _ => Ok("data", ("Cache-Control", "public, max-age=60")); + await cache.SendAsync(Get("https://a.example/b", ("Authorization", "Bearer x")), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://a.example/b", ("Authorization", "Bearer x")), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(3, server.Requests.Count); + } + + [Test] + public async Task StaleResponseIsRevalidatedWithValidators() + { + var server = new Server(_ => Ok("original", ("Cache-Control", "max-age=10"), ("ETag", "\"v1\""), ("Last-Modified", "Sat, 01 Aug 2026 10:00:00 GMT"))); + var cache = CreateCache(out var clock); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + clock.Advance(TimeSpan.FromSeconds(20)); + server.Respond = _ => Response(HttpStatusCode.NotModified, String.Empty, ("Cache-Control", "max-age=100"), ("ETag", "\"v1\"")); + var revalidated = await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + clock.Advance(TimeSpan.FromSeconds(50)); + var fresh = await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(2, server.Requests.Count); + Assert.AreEqual("\"v1\"", server.Requests[1].Headers[HeaderNames.IfNoneMatch]); + Assert.AreEqual("Sat, 01 Aug 2026 10:00:00 GMT", server.Requests[1].Headers[HeaderNames.IfModifiedSince]); + Assert.AreEqual(HttpStatusCode.OK, revalidated.StatusCode); + Assert.AreEqual("original", Read(revalidated)); + Assert.AreEqual("max-age=100", revalidated.Headers[HeaderNames.CacheControl]); + Assert.AreEqual("original", Read(fresh)); + } + + [Test] + public async Task ChangedResponseReplacesStoredOneOnRevalidation() + { + var server = new Server(_ => Ok("v1", ("Cache-Control", "max-age=0"), ("ETag", "\"v1\""))); + var cache = CreateCache(out _); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + server.Respond = _ => Ok("v2", ("Cache-Control", "max-age=60"), ("ETag", "\"v2\"")); + var changed = await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + var stored = await cache.MatchAsync(Get("https://a.example/page"), CancellationToken.None); + + Assert.AreEqual("v2", Read(changed)); + Assert.AreEqual("v2", Read(stored)); + } + + [Test] + public async Task ConditionalRequestOfCallerIsPassedThrough() + { + var server = new Server(_ => Ok("v1", ("Cache-Control", "max-age=0"), ("ETag", "\"v1\""))); + var cache = CreateCache(out _); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + server.Respond = _ => Response(HttpStatusCode.NotModified, String.Empty); + var response = await cache.SendAsync(Get("https://a.example/page", ("If-None-Match", "\"other\"")), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(HttpStatusCode.NotModified, response.StatusCode); + Assert.AreEqual("\"other\"", server.Requests[1].Headers[HeaderNames.IfNoneMatch]); + } + + [Test] + public async Task HeuristicFreshnessUsesLastModified() + { + // the Date is 10 days after Last-Modified, i.e., fresh for 1 day + var server = new Server(_ => Ok("page", ("Date", "Mon, 19 Oct 2026 12:00:00 GMT"), ("Last-Modified", "Fri, 09 Oct 2026 12:00:00 GMT"))); + var cache = CreateCache(out var clock); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + clock.Advance(TimeSpan.FromHours(23)); + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(1, server.Requests.Count); + + clock.Advance(TimeSpan.FromHours(2)); + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(2, server.Requests.Count); + Assert.AreEqual("Fri, 09 Oct 2026 12:00:00 GMT", server.Requests[1].Headers[HeaderNames.IfModifiedSince]); + } + + [Test] + public async Task HeuristicFreshnessCanBeDisabled() + { + var server = new Server(_ => Ok("page", ("Date", "Mon, 19 Oct 2026 12:00:00 GMT"), ("Last-Modified", "Fri, 09 Oct 2026 12:00:00 GMT"))); + var cache = CreateCache(out _, new HttpCacheOptions { UseHeuristicFreshness = false }); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(2, server.Requests.Count); + } + + [Test] + public async Task ExpiresHeaderIsRespected() + { + var server = new Server(_ => Ok("page", ("Date", "Mon, 19 Oct 2026 12:00:00 GMT"), ("Expires", "Mon, 19 Oct 2026 12:01:00 GMT"))); + var cache = CreateCache(out var clock); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + clock.Advance(TimeSpan.FromSeconds(59)); + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + clock.Advance(TimeSpan.FromSeconds(2)); + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(2, server.Requests.Count); + } + + [Test] + public async Task VaryKeepsSeparateVariants() + { + var server = new Server(request => Ok(request.Headers["Accept-Language"], ("Cache-Control", "max-age=60"), ("Vary", "Accept-Language"))); + var cache = CreateCache(out _); + + await cache.SendAsync(Get("https://a.example/page", ("Accept-Language", "en")), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://a.example/page", ("Accept-Language", "de")), server.SendAsync, CancellationToken.None); + var en = await cache.SendAsync(Get("https://a.example/page", ("Accept-Language", "en")), server.SendAsync, CancellationToken.None); + var de = await cache.SendAsync(Get("https://a.example/page", ("Accept-Language", "de")), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(2, server.Requests.Count); + Assert.AreEqual("en", Read(en)); + Assert.AreEqual("de", Read(de)); + } + + [Test] + public async Task VaryStarIsNotStored() + { + var server = new Server(_ => Ok("page", ("Cache-Control", "max-age=60"), ("Vary", "*"))); + var cache = CreateCache(out _); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(2, server.Requests.Count); + } + + [Test] + public async Task SetCookieIsNotReplayedFromCache() + { + var server = new Server(_ => Ok("page", ("Cache-Control", "max-age=60"), ("Set-Cookie", "a=b"))); + var cache = CreateCache(out _); + + var first = await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + var second = await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual("a=b", first.Headers[HeaderNames.SetCookie]); + Assert.IsFalse(second.Headers.ContainsKey(HeaderNames.SetCookie)); + } + + [Test] + public async Task RequestNoCacheForcesRevalidation() + { + var server = new Server(_ => Ok("page", ("Cache-Control", "max-age=60"), ("ETag", "\"v1\""))); + var cache = CreateCache(out _); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://a.example/page", ("Cache-Control", "no-cache")), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://a.example/page", ("Pragma", "no-cache")), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(3, server.Requests.Count); + Assert.AreEqual("\"v1\"", server.Requests[1].Headers[HeaderNames.IfNoneMatch]); + } + + [Test] + public async Task RequestMaxStaleAcceptsStaleResponse() + { + var server = new Server(_ => Ok("page", ("Cache-Control", "max-age=10"))); + var cache = CreateCache(out var clock); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + clock.Advance(TimeSpan.FromSeconds(30)); + await cache.SendAsync(Get("https://a.example/page", ("Cache-Control", "max-stale=60")), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(1, server.Requests.Count); + } + + [Test] + public async Task OnlyIfCachedAnswersWithGatewayTimeoutOnMiss() + { + var server = new Server(_ => Ok("page", ("Cache-Control", "max-age=60"))); + var cache = CreateCache(out _, new HttpCacheOptions { Mode = HttpCacheMode.OnlyIfCached }); + + var response = await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(HttpStatusCode.GatewayTimeout, response.StatusCode); + Assert.AreEqual(0, server.Requests.Count); + } + + [Test] + public async Task ForceCacheServesStaleResponses() + { + var server = new Server(_ => Ok("page", ("Cache-Control", "max-age=1"))); + var cache = CreateCache(out var clock, new HttpCacheOptions { Mode = HttpCacheMode.ForceCache }); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + clock.Advance(TimeSpan.FromDays(30)); + var response = await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(1, server.Requests.Count); + Assert.AreEqual("page", Read(response)); + } + + [Test] + public async Task ReloadAlwaysFetchesButStores() + { + var server = new Server(_ => Ok("page", ("Cache-Control", "max-age=60"))); + var cache = CreateCache(out _, new HttpCacheOptions { Mode = HttpCacheMode.Reload }); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(2, server.Requests.Count); + Assert.IsNotNull(await cache.MatchAsync(Get("https://a.example/page"), CancellationToken.None)); + } + + [Test] + public async Task NetworkFailureServesStaleResponse() + { + var server = new Server(_ => Ok("page", ("Cache-Control", "max-age=10"))); + var cache = CreateCache(out var clock); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + clock.Advance(TimeSpan.FromSeconds(20)); + server.Respond = request => new DefaultResponse { Address = request.Address, StatusCode = 0 }; + var response = await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + Assert.AreEqual("page", Read(response)); + } + + [Test] + public async Task HeadIsAnsweredFromStoredGet() + { + var server = new Server(_ => Ok("page", ("Cache-Control", "max-age=60"))); + var cache = CreateCache(out _); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + var response = await cache.SendAsync(Create(HttpMethod.Head, "https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(1, server.Requests.Count); + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + Assert.AreEqual(String.Empty, Read(response)); + } + + [Test] + public async Task LargeResponseIsPassedThroughCompletely() + { + var text = new String('x', 1000); + var server = new Server(_ => Ok(text, ("Cache-Control", "max-age=60"))); + var cache = CreateCache(out _, new HttpCacheOptions { MaxEntrySize = 100 }); + + var response = await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(text, Read(response)); + Assert.IsNull(await cache.MatchAsync(Get("https://a.example/page"), CancellationToken.None)); + } + + [Test] + public async Task NonHttpRequestsBypassTheCache() + { + var server = new Server(_ => Ok("file", ("Cache-Control", "max-age=60"))); + var cache = CreateCache(out _); + + await cache.SendAsync(Get("file:///tmp/a.txt"), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("file:///tmp/a.txt"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(2, server.Requests.Count); + } + + [Test] + public async Task BypassPredicateExcludesRequests() + { + var server = new Server(_ => Ok("page", ("Cache-Control", "max-age=60"))); + var cache = CreateCache(out _, new HttpCacheOptions { Bypass = request => request.Address.Path.StartsWith("api") }); + + await cache.SendAsync(Get("https://a.example/api/x"), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://a.example/api/x"), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(2, server.Requests.Count); + } + + [Test] + public async Task EntriesAreKeptPerOrigin() + { + var handler = new MemoryHttpCacheHandler(); + var server = new Server(_ => Ok("page", ("Cache-Control", "max-age=60"))); + var cache = CreateCache(out _, new HttpCacheOptions { Handler = handler }); + + await cache.SendAsync(Get("https://a.example/page"), server.SendAsync, CancellationToken.None); + await cache.SendAsync(Get("https://b.example/page"), server.SendAsync, CancellationToken.None); + + CollectionAssert.AreEquivalent(new[] { "https://a.example", "https://b.example" }, handler.GetOrigins().ToArray()); + + await cache.ClearAsync("https://a.example", CancellationToken.None); + + Assert.IsNull(await cache.MatchAsync(Get("https://a.example/page"), CancellationToken.None)); + Assert.IsNotNull(await cache.MatchAsync(Get("https://b.example/page"), CancellationToken.None)); + + await cache.ClearAsync(CancellationToken.None); + + Assert.IsEmpty(handler.GetOrigins()); + } + + [Test] + public async Task PersistedEntriesSurviveANewCache() + { + var directory = Path.Combine(Path.GetTempPath(), "anglesharp-http-cache-" + Guid.NewGuid().ToString("N")); + + try + { + var server = new Server(_ => Ok("persisted", ("Cache-Control", "max-age=60"), ("Vary", "Accept"), ("Set-Cookie", "a=b"))); + var first = CreateCache(out _, new HttpCacheOptions { Handler = new PersistenceHttpCacheHandler(directory) }); + await first.SendAsync(Get("https://a.example/page", ("Accept", "text/html")), server.SendAsync, CancellationToken.None); + + var handler = new PersistenceHttpCacheHandler(directory); + var second = CreateCache(out _, new HttpCacheOptions { Handler = handler }); + var response = await second.SendAsync(Get("https://a.example/page", ("Accept", "text/html")), server.SendAsync, CancellationToken.None); + + Assert.AreEqual(1, server.Requests.Count); + Assert.AreEqual("persisted", Read(response)); + Assert.IsFalse(response.Headers.ContainsKey(HeaderNames.SetCookie)); + CollectionAssert.AreEqual(new[] { "https://a.example" }, handler.GetOrigins().ToArray()); + CollectionAssert.AreEqual(new[] { "https://a.example/page" }, handler.GetKeys("https://a.example").ToArray()); + + await second.ClearAsync(CancellationToken.None); + + Assert.IsEmpty(handler.GetOrigins()); + } + finally + { + if (Directory.Exists(directory)) + { + Directory.Delete(directory, true); + } + } + } + + [Test] + public void WithHttpCacheWrapsHttpRequestersOnly() + { + var config = Configuration.Default.WithRequesters().WithHttpCache(); + var requesters = config.Services.OfType().ToArray(); + + Assert.IsInstanceOf(requesters[0]); + Assert.IsInstanceOf(((CachingRequester)requesters[0]).Inner); + Assert.AreEqual(1, requesters.OfType().Count()); + Assert.AreEqual(1, config.Services.OfType().Count()); + } + + [Test] + public void WithHttpCacheTwiceDoesNotWrapTwice() + { + var cache = new HttpCache(); + var config = Configuration.Default.WithRequesters().WithHttpCache().WithHttpCache(cache); + var caching = config.Services.OfType().Single(); + + Assert.AreSame(cache, caching.Cache); + Assert.IsInstanceOf(caching.Inner); + Assert.AreSame(cache, config.Services.OfType().Single()); + } + + [Test] + public void WithIoWithoutHttpCacheOptionsDoesNotWrap() + { + var config = Configuration.Default.WithIo(new IoOptions()); + + Assert.IsEmpty(config.Services.OfType()); + } + + [Test] + public void TemporaryIoOptionsEnableTheHttpCache() + { + var config = Configuration.Default.WithIo(IoOptions.CreateTemporary()); + + Assert.AreEqual(1, config.Services.OfType().Count()); + } + + [Test] + public void PersistentIoOptionsShareOneRootDirectory() + { + var directory = Path.Combine(Path.GetTempPath(), "anglesharp-io-" + Guid.NewGuid().ToString("N")); + + try + { + var options = IoOptions.CreatePersistent(directory); + + Assert.IsInstanceOf(options.HttpCache.Handler); + Assert.IsTrue(Directory.Exists(Path.Combine(directory, "http-cache"))); + } + finally + { + if (Directory.Exists(directory)) + { + Directory.Delete(directory, true); + } + } + } + + [Test] + public async Task DocumentLoadsAreCachedThroughConfiguration() + { + var requester = new MockRequester(); + var count = 0; + requester.OnRequest = _ => count++; + requester.BuildResponse(request => new DefaultResponse + { + Address = request.Address, + StatusCode = HttpStatusCode.OK, + Headers = new Dictionary + { + { "Cache-Control", "max-age=60" }, + { "Content-Type", "text/html" }, + }, + Content = new MemoryStream(Encoding.UTF8.GetBytes("Cached")), + }); + var config = Configuration.Default.WithDefaultLoader().WithRequester(requester).WithHttpCache(); + var context = BrowsingContext.New(config); + + var first = await context.OpenAsync("https://a.example/"); + var second = await context.OpenAsync("https://a.example/"); + + Assert.AreEqual(1, count); + Assert.AreEqual("Cached", first.Title); + Assert.AreEqual("Cached", second.Title); + } + + private static HttpCache CreateCache(out TestClock clock, HttpCacheOptions options = null) + { + var current = new TestClock(Start); + clock = current; + return new HttpCache(options ?? new HttpCacheOptions()) + { + Clock = () => current.Now, + }; + } + + private static Request Get(String address, params (String, String)[] headers) => + Create(HttpMethod.Get, address, headers); + + private static Request Create(HttpMethod method, String address, params (String, String)[] headers) + { + var request = new Request + { + Method = method, + Address = new Url(address), + }; + + foreach (var (name, value) in headers) + { + request.Headers[name] = value; + } + + return request; + } + + private static DefaultResponse Ok(String content, params (String, String)[] headers) => + Response(HttpStatusCode.OK, content, headers); + + private static DefaultResponse Response(HttpStatusCode status, String content, params (String, String)[] headers) + { + // HttpClientRequester creates case-sensitive dictionaries, too + var response = new DefaultResponse + { + StatusCode = status, + Headers = new Dictionary(), + Content = new MemoryStream(Encoding.UTF8.GetBytes(content)), + }; + + foreach (var (name, value) in headers) + { + response.Headers[name] = value; + } + + return response; + } + + private static String Read(IResponse response) + { + using (var reader = new StreamReader(response.Content)) + { + return reader.ReadToEnd(); + } + } + + private sealed class TestClock + { + public TestClock(DateTimeOffset now) => Now = now; + + public DateTimeOffset Now { get; private set; } + + public void Advance(TimeSpan time) => Now += time; + } + + private sealed class Server + { + public Server(Func respond) => Respond = respond; + + public Func Respond { get; set; } + + public List Requests { get; } = new List(); + + public Task SendAsync(Request request, CancellationToken cancel) + { + Requests.Add(request); + var response = Respond.Invoke(request); + response.Address = response.Address ?? request.Address; + return Task.FromResult(response); + } + } + } +} diff --git a/src/AngleSharp.Io/Http/CacheControl.cs b/src/AngleSharp.Io/Http/CacheControl.cs new file mode 100644 index 0000000..28e6daf --- /dev/null +++ b/src/AngleSharp.Io/Http/CacheControl.cs @@ -0,0 +1,147 @@ +namespace AngleSharp.Io.Http +{ + using System; + using System.Collections.Generic; + using System.Globalization; + + /// + /// The parsed directives of a Cache-Control (and Pragma) header. + /// + internal sealed class CacheControl + { + public Boolean NoStore { get; private set; } + + public Boolean NoCache { get; private set; } + + public Boolean MustRevalidate { get; private set; } + + public Boolean Public { get; private set; } + + public Boolean OnlyIfCached { get; private set; } + + public Boolean MaxStaleAny { get; private set; } + + public TimeSpan? MaxAge { get; private set; } + + public TimeSpan? SMaxAge { get; private set; } + + public TimeSpan? MinFresh { get; private set; } + + public TimeSpan? MaxStale { get; private set; } + + public static CacheControl Parse(IDictionary headers) + { + var result = new CacheControl(); + var value = HttpCachePolicy.GetHeader(headers, HeaderNames.CacheControl); + + if (!String.IsNullOrEmpty(value)) + { + foreach (var directive in Split(value)) + { + result.Apply(directive.Key, directive.Value); + } + } + else + { + // Pragma is only considered without Cache-Control (RFC 9111, 5.4) + var pragma = HttpCachePolicy.GetHeader(headers, HeaderNames.Pragma); + + if (pragma != null && pragma.IndexOf("no-cache", StringComparison.OrdinalIgnoreCase) >= 0) + { + result.NoCache = true; + } + } + + return result; + } + + private void Apply(String name, String value) + { + switch (name.ToLowerInvariant()) + { + case "no-store": + NoStore = true; + break; + case "no-cache": + // the qualified form (no-cache="field") is treated like the + // unqualified one, which is the conservative choice + NoCache = true; + break; + case "must-revalidate": + case "proxy-revalidate": + MustRevalidate = true; + break; + case "public": + Public = true; + break; + case "only-if-cached": + OnlyIfCached = true; + break; + case "max-age": + MaxAge = ParseSeconds(value) ?? TimeSpan.Zero; + break; + case "s-maxage": + SMaxAge = ParseSeconds(value) ?? TimeSpan.Zero; + break; + case "min-fresh": + MinFresh = ParseSeconds(value); + break; + case "max-stale": + MaxStale = ParseSeconds(value); + MaxStaleAny = value == null; + break; + } + } + + private static TimeSpan? ParseSeconds(String value) + { + if (value != null && Int64.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var seconds)) + { + return TimeSpan.FromSeconds(Math.Min(seconds, Int32.MaxValue)); + } + + return null; + } + + private static IEnumerable> Split(String value) + { + var index = 0; + + while (index < value.Length) + { + var start = index; + var quoted = false; + + // commas inside quoted values (no-cache="a, b") do not separate + while (index < value.Length && (quoted || value[index] != ',')) + { + if (value[index] == '"') + { + quoted = !quoted; + } + + index++; + } + + var part = value.Substring(start, index - start).Trim(); + index++; + + if (part.Length > 0) + { + var eq = part.IndexOf('='); + + if (eq < 0) + { + yield return new KeyValuePair(part, null); + } + else + { + var name = part.Substring(0, eq).Trim(); + var argument = part.Substring(eq + 1).Trim().Trim('"'); + yield return new KeyValuePair(name, argument); + } + } + } + } + } +} diff --git a/src/AngleSharp.Io/Http/CachingRequester.cs b/src/AngleSharp.Io/Http/CachingRequester.cs new file mode 100644 index 0000000..a7cdbc1 --- /dev/null +++ b/src/AngleSharp.Io/Http/CachingRequester.cs @@ -0,0 +1,69 @@ +namespace AngleSharp.Io.Http +{ + using System; + using System.Threading; + using System.Threading.Tasks; + + /// + /// A requester answering HTTP(S) requests from an HTTP cache before + /// delegating to the wrapped requester. + /// + public class CachingRequester : BaseRequester + { + #region Fields + + private readonly IRequester _inner; + private readonly IHttpCache _cache; + + #endregion + + #region ctor + + /// + /// Creates a new caching requester. + /// + /// The requester used for network requests. + /// The cache to use. + public CachingRequester(IRequester inner, IHttpCache cache) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + } + + #endregion + + #region Properties + + /// + /// Gets the wrapped requester. + /// + public IRequester Inner => _inner; + + /// + /// Gets the used cache. + /// + public IHttpCache Cache => _cache; + + #endregion + + #region Methods + + /// + /// Checks if the given protocol is supported by the wrapped requester. + /// + /// The protocol to check for, e.g., http. + /// True if the protocol is supported, otherwise false. + public override Boolean SupportsProtocol(String protocol) => _inner.SupportsProtocol(protocol); + + /// + /// Performs the request using the cache. + /// + /// The options to consider. + /// The token for cancelling the task. + /// The task that will eventually give the response data. + protected override Task PerformRequestAsync(Request request, CancellationToken cancel) => + _cache.SendAsync(request, _inner.RequestAsync, cancel); + + #endregion + } +} diff --git a/src/AngleSharp.Io/Http/HttpCache.cs b/src/AngleSharp.Io/Http/HttpCache.cs new file mode 100644 index 0000000..1923afb --- /dev/null +++ b/src/AngleSharp.Io/Http/HttpCache.cs @@ -0,0 +1,386 @@ +namespace AngleSharp.Io.Http +{ + using AngleSharp.Dom; + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Threading; + using System.Threading.Tasks; + using HttpMethod = AngleSharp.Io.HttpMethod; + + /// + /// Represents a private HTTP cache following RFC 9111. Entries are kept + /// per origin of the requested address. + /// + public class HttpCache : IHttpCache + { + #region Fields + + private readonly HttpCacheOptions _options; + + #endregion + + #region ctor + + /// + /// Creates a new HTTP cache keeping its entries in memory. + /// + public HttpCache() + : this(new HttpCacheOptions()) + { + } + + /// + /// Creates a new HTTP cache with the given options. + /// + /// The options to use. + public HttpCache(HttpCacheOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + + if (_options.Handler == null) + { + throw new ArgumentException("A handler for storing the cache entries is required.", nameof(options)); + } + } + + #endregion + + #region Properties + + /// + public HttpCacheOptions Options => _options; + + internal Func Clock { get; set; } = () => DateTimeOffset.UtcNow; + + private IHttpCacheHandler Handler => _options.Handler; + + #endregion + + #region Methods + + /// + public async Task SendAsync(Request request, Func> network, CancellationToken cancel) + { + var address = request.Address; + + if (address == null || !IsHttp(address.Scheme) || (_options.Bypass?.Invoke(request) ?? false)) + { + return await network.Invoke(request, cancel).ConfigureAwait(false); + } + + if (request.Method != HttpMethod.Get && request.Method != HttpMethod.Head) + { + var result = await network.Invoke(request, cancel).ConfigureAwait(false); + + if (IsUnsafe(request.Method) && result != null) + { + await InvalidateAsync(address, result, cancel).ConfigureAwait(false); + } + + return result; + } + + var requestControl = CacheControl.Parse(request.Headers); + var mode = GetMode(requestControl); + + if (mode == HttpCacheMode.NoStore) + { + return await network.Invoke(request, cancel).ConfigureAwait(false); + } + + var origin = GetOrigin(address); + var key = GetKey(address); + var variants = await Handler.ReadAsync(origin, key, cancel).ConfigureAwait(false); + var stored = mode != HttpCacheMode.Reload ? variants.FirstOrDefault(m => HttpCachePolicy.MatchesVary(m, request)) : null; + + if (stored != null) + { + var now = Clock.Invoke(); + + if (mode == HttpCacheMode.ForceCache || mode == HttpCacheMode.OnlyIfCached || + (mode == HttpCacheMode.Default && HttpCachePolicy.IsFreshEnough(stored, requestControl, now, _options.UseHeuristicFreshness))) + { + return CreateResponse(stored, request.Method, now); + } + } + else if (mode == HttpCacheMode.OnlyIfCached) + { + return new DefaultResponse + { + Address = address, + StatusCode = HttpStatusCode.GatewayTimeout, + Content = Stream.Null, + }; + } + + // a request carrying its own validators is answered by the server + var validating = stored != null && HttpCachePolicy.HasValidator(stored) && !HttpCachePolicy.IsConditional(request); + var networkRequest = validating ? CreateConditionalRequest(request, stored) : request; + var requestTime = Clock.Invoke(); + var response = await network.Invoke(networkRequest, cancel).ConfigureAwait(false); + var responseTime = Clock.Invoke(); + + if (response == null) + { + return null; + } + + if (validating && response.StatusCode == HttpStatusCode.NotModified) + { + var updated = HttpCachePolicy.Update(stored, response.Headers, requestTime, responseTime); + response.Dispose(); + await StoreAsync(origin, key, variants, request, updated, cancel).ConfigureAwait(false); + return CreateResponse(updated, request.Method, responseTime); + } + + // the requesters report network failures with status 0 (#28), in + // which case a stale response is better than none (RFC 9111, 4.2.4) + if (stored != null && response.StatusCode == 0 && HttpCachePolicy.MayServeStale(stored)) + { + response.Dispose(); + return CreateResponse(stored, request.Method, responseTime); + } + + if (request.Method != HttpMethod.Get || requestControl.NoStore || !HttpCachePolicy.IsStorable(request, response.Headers, response.StatusCode)) + { + return response; + } + + return await BufferAndStoreAsync(request, response, origin, key, variants, requestTime, responseTime, cancel).ConfigureAwait(false); + } + + /// + public async Task MatchAsync(Request request, CancellationToken cancel) + { + var address = request.Address; + + if (address == null || !IsHttp(address.Scheme)) + { + return null; + } + + var variants = await Handler.ReadAsync(GetOrigin(address), GetKey(address), cancel).ConfigureAwait(false); + var stored = variants.FirstOrDefault(m => HttpCachePolicy.MatchesVary(m, request)); + return stored != null ? CreateResponse(stored, request.Method, Clock.Invoke()) : null; + } + + /// + public async Task RemoveAsync(Url address, CancellationToken cancel) + { + var origin = GetOrigin(address); + var key = GetKey(address); + var variants = await Handler.ReadAsync(origin, key, cancel).ConfigureAwait(false); + + if (variants.Count > 0) + { + await Handler.WriteAsync(origin, key, null, cancel).ConfigureAwait(false); + return true; + } + + return false; + } + + /// + public async Task ClearAsync(String origin, CancellationToken cancel) + { + foreach (var key in Handler.GetKeys(origin).ToArray()) + { + await Handler.WriteAsync(origin, key, null, cancel).ConfigureAwait(false); + } + } + + /// + public async Task ClearAsync(CancellationToken cancel) + { + foreach (var origin in Handler.GetOrigins().ToArray()) + { + await ClearAsync(origin, cancel).ConfigureAwait(false); + } + } + + #endregion + + #region Helpers + + private HttpCacheMode GetMode(CacheControl requestControl) + { + var mode = _options.Mode; + + // the request directives only refine the default behavior, just + // like the Fetch specification does for the default cache mode + if (mode == HttpCacheMode.Default) + { + if (requestControl.NoStore) + { + return HttpCacheMode.NoStore; + } + else if (requestControl.OnlyIfCached) + { + return HttpCacheMode.OnlyIfCached; + } + else if (requestControl.NoCache) + { + return HttpCacheMode.NoCache; + } + } + + return mode; + } + + private async Task BufferAndStoreAsync(Request request, IResponse response, String origin, String key, IReadOnlyList variants, DateTimeOffset requestTime, DateTimeOffset responseTime, CancellationToken cancel) + { + var content = response.Content; + var limit = _options.MaxEntrySize; + var body = Array.Empty(); + + if (content != null) + { + var length = HttpCachePolicy.GetHeader(response.Headers, HeaderNames.ContentLength); + + if (Int64.TryParse(length, out var size) && size > limit) + { + return response; + } + + var buffer = new MemoryStream(); + var chunk = new Byte[81920]; + + while (true) + { + var read = await content.ReadAsync(chunk, 0, chunk.Length, cancel).ConfigureAwait(false); + + if (read == 0) + { + break; + } + + buffer.Write(chunk, 0, read); + + if (buffer.Length > limit) + { + return Copy(response, new PrefixedStream(buffer.ToArray(), content)); + } + } + + content.Dispose(); + body = buffer.ToArray(); + } + + var entry = HttpCachePolicy.CreateEntry(request, response, body, requestTime, responseTime); + await StoreAsync(origin, key, variants, request, entry, cancel).ConfigureAwait(false); + + // the original headers are kept, such that the loader still sees + // the Set-Cookie header of the network response + return Copy(response, new MemoryStream(body, false)); + } + + private Task StoreAsync(String origin, String key, IReadOnlyList variants, Request request, HttpCacheEntry entry, CancellationToken cancel) + { + // the new entry replaces all variants the request would have selected + var remaining = variants.Where(m => !HttpCachePolicy.MatchesVary(m, request)); + return Handler.WriteAsync(origin, key, remaining.Concat(new[] { entry }).ToArray(), cancel); + } + + private async Task InvalidateAsync(Url address, IResponse response, CancellationToken cancel) + { + var status = (Int32)response.StatusCode; + + if (status >= 200 && status < 400) + { + await RemoveAsync(address, cancel).ConfigureAwait(false); + + foreach (var name in new[] { HeaderNames.Location, HeaderNames.ContentLocation }) + { + var value = HttpCachePolicy.GetHeader(response.Headers, name); + + if (!String.IsNullOrEmpty(value)) + { + var target = new Url(address, value); + + // only same-origin targets may be invalidated (RFC 9111, 4.4) + if (!target.IsInvalid && GetOrigin(target) == GetOrigin(address)) + { + await RemoveAsync(target, cancel).ConfigureAwait(false); + } + } + } + } + } + + private static Request CreateConditionalRequest(Request request, HttpCacheEntry stored) + { + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var header in request.Headers) + { + headers[header.Key] = header.Value; + } + + var etag = HttpCachePolicy.GetHeader(stored.ResponseHeaders, HeaderNames.ETag); + var lastModified = HttpCachePolicy.GetHeader(stored.ResponseHeaders, HeaderNames.LastModified); + + if (etag != null) + { + headers[HeaderNames.IfNoneMatch] = etag; + } + + if (lastModified != null) + { + headers[HeaderNames.IfModifiedSince] = lastModified; + } + + return new Request + { + Method = request.Method, + Address = request.Address, + Content = request.Content, + Headers = headers, + }; + } + + private static IResponse CreateResponse(HttpCacheEntry entry, HttpMethod method, DateTimeOffset now) + { + var headers = new Dictionary(entry.ResponseHeaders, StringComparer.OrdinalIgnoreCase); + var age = HttpCachePolicy.GetCurrentAge(entry, now); + headers[HeaderNames.Age] = ((Int64)Math.Max(0, age.TotalSeconds)).ToString(System.Globalization.CultureInfo.InvariantCulture); + + return new DefaultResponse + { + Address = new Url(entry.Address), + StatusCode = entry.StatusCode, + Headers = headers, + Content = method == HttpMethod.Head ? Stream.Null : new MemoryStream(entry.Body, false), + }; + } + + private static IResponse Copy(IResponse response, Stream content) => new DefaultResponse + { + Address = response.Address, + StatusCode = response.StatusCode, + Headers = response.Headers, + Content = content, + }; + + private static Boolean IsHttp(String scheme) => + scheme.Equals(ProtocolNames.Http, StringComparison.OrdinalIgnoreCase) || + scheme.Equals(ProtocolNames.Https, StringComparison.OrdinalIgnoreCase); + + private static Boolean IsUnsafe(HttpMethod method) => + method != HttpMethod.Get && method != HttpMethod.Head && + method != HttpMethod.Options && method != HttpMethod.Trace; + + private static String GetOrigin(Url address) => address.Origin ?? String.Empty; + + private static String GetKey(Url address) + { + var href = address.Href; + var index = href.IndexOf('#'); + return index >= 0 ? href.Substring(0, index) : href; + } + + #endregion + } +} diff --git a/src/AngleSharp.Io/Http/HttpCacheEntry.cs b/src/AngleSharp.Io/Http/HttpCacheEntry.cs new file mode 100644 index 0000000..8cc6fbd --- /dev/null +++ b/src/AngleSharp.Io/Http/HttpCacheEntry.cs @@ -0,0 +1,57 @@ +namespace AngleSharp.Io.Http +{ + using System; + using System.Collections.Generic; + using System.Net; + + /// + /// Represents a stored response of the HTTP cache. + /// + public sealed class HttpCacheEntry + { + /// + /// Creates a new HTTP cache entry. + /// + public HttpCacheEntry() + { + RequestHeaders = new Dictionary(StringComparer.OrdinalIgnoreCase); + ResponseHeaders = new Dictionary(StringComparer.OrdinalIgnoreCase); + Body = Array.Empty(); + } + + /// + /// Gets or sets the absolute address of the stored response. + /// + public String Address { get; set; } + + /// + /// Gets or sets the status code of the stored response. + /// + public HttpStatusCode StatusCode { get; set; } + + /// + /// Gets the request headers selected by the Vary header of the response. + /// + public IDictionary RequestHeaders { get; } + + /// + /// Gets the stored response headers. + /// + public IDictionary ResponseHeaders { get; } + + /// + /// Gets or sets the time when the request has been sent. + /// + public DateTimeOffset RequestTime { get; set; } + + /// + /// Gets or sets the time when the response has been received. + /// + public DateTimeOffset ResponseTime { get; set; } + + /// + /// Gets or sets the stored response body. + /// + public Byte[] Body { get; set; } + } +} diff --git a/src/AngleSharp.Io/Http/HttpCacheMode.cs b/src/AngleSharp.Io/Http/HttpCacheMode.cs new file mode 100644 index 0000000..7d83db5 --- /dev/null +++ b/src/AngleSharp.Io/Http/HttpCacheMode.cs @@ -0,0 +1,37 @@ +namespace AngleSharp.Io.Http +{ + /// + /// The modes of the HTTP cache. The names follow the RequestCache values + /// of the Fetch specification. + /// + public enum HttpCacheMode + { + /// + /// Uses fresh responses from the cache, revalidates stale ones, and + /// stores new responses. + /// + Default, + /// + /// Bypasses the cache completely, i.e., neither reads nor writes. + /// + NoStore, + /// + /// Always goes to the network, but stores the response. + /// + Reload, + /// + /// Always revalidates stored responses with the server. + /// + NoCache, + /// + /// Uses any stored response, even if stale. Goes to the network only + /// if nothing is stored. + /// + ForceCache, + /// + /// Uses any stored response, even if stale. Never goes to the network; + /// answers with a 504 if nothing is stored. + /// + OnlyIfCached, + } +} diff --git a/src/AngleSharp.Io/Http/HttpCacheOptions.cs b/src/AngleSharp.Io/Http/HttpCacheOptions.cs new file mode 100644 index 0000000..a0b104c --- /dev/null +++ b/src/AngleSharp.Io/Http/HttpCacheOptions.cs @@ -0,0 +1,40 @@ +namespace AngleSharp.Io.Http +{ + using System; + + /// + /// Options for setting up the HTTP cache. + /// + public sealed class HttpCacheOptions + { + /// + /// Gets or sets the handler storing the cache entries. By default, the + /// entries are only kept in memory. + /// + public IHttpCacheHandler Handler { get; set; } = new MemoryHttpCacheHandler(); + + /// + /// Gets or sets the mode of the cache. + /// + public HttpCacheMode Mode { get; set; } = HttpCacheMode.Default; + + /// + /// Gets or sets the maximum size of a response body in bytes to be + /// stored. Larger responses are passed through without being stored. + /// + public Int64 MaxEntrySize { get; set; } = 10 * 1024 * 1024; + + /// + /// Gets or sets if responses without explicit expiration, but with a + /// Last-Modified header, are considered fresh for a fraction of their + /// age (RFC 9111, 4.2.2). + /// + public Boolean UseHeuristicFreshness { get; set; } = true; + + /// + /// Gets or sets a predicate to exclude requests from the cache. Returning + /// true for a request bypasses the cache for it. + /// + public Func Bypass { get; set; } + } +} diff --git a/src/AngleSharp.Io/Http/HttpCachePolicy.cs b/src/AngleSharp.Io/Http/HttpCachePolicy.cs new file mode 100644 index 0000000..fbe7a7c --- /dev/null +++ b/src/AngleSharp.Io/Http/HttpCachePolicy.cs @@ -0,0 +1,317 @@ +namespace AngleSharp.Io.Http +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Linq; + using System.Net; + + /// + /// The storage and freshness rules of a private cache (RFC 9111). + /// + internal static class HttpCachePolicy + { + private static readonly TimeSpan MaxHeuristicLifetime = TimeSpan.FromDays(7); + + private static readonly String[] DateFormats = new[] + { + "r", + "dddd, dd-MMM-yy HH:mm:ss 'GMT'", + "ddd MMM d HH:mm:ss yyyy", + }; + + private static readonly HashSet HeuristicallyCacheable = new HashSet + { + 200, 203, 204, 300, 301, 308, 404, 405, 410, 414, 501, + }; + + // never stored, since cookies are applied by the loader on every + // response it sees, including those served from the cache + private static readonly String[] UnstoredHeaders = new[] + { + HeaderNames.SetCookie, + "Set-Cookie2", + }; + + // not taken from a 304 when updating a stored response (RFC 9111, 3.2) + private static readonly String[] UnmergedHeaders = new[] + { + HeaderNames.ContentLength, + HeaderNames.SetCookie, + "Set-Cookie2", + "Transfer-Encoding", + HeaderNames.Connection, + "Keep-Alive", + }; + + public static String GetHeader(IDictionary headers, String name) + { + if (headers == null) + { + return null; + } + + if (headers.TryGetValue(name, out var value)) + { + return value; + } + + // redirected requests and HttpClient responses may come with + // case-sensitive dictionaries + foreach (var header in headers) + { + if (header.Key.Equals(name, StringComparison.OrdinalIgnoreCase)) + { + return header.Value; + } + } + + return null; + } + + public static Boolean TryParseDate(String value, out DateTimeOffset result) => + DateTimeOffset.TryParseExact(value?.Trim(), DateFormats, CultureInfo.InvariantCulture, + DateTimeStyles.AllowWhiteSpaces | DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out result); + + public static Boolean IsStorable(Request request, IDictionary responseHeaders, HttpStatusCode status) + { + var code = (Int32)status; + + if (code < 200 || status == HttpStatusCode.NotModified || status == HttpStatusCode.PartialContent) + { + return false; + } + + var control = CacheControl.Parse(responseHeaders); + + if (control.NoStore || GetVary(responseHeaders).Contains("*")) + { + return false; + } + + if (GetHeader(request.Headers, HeaderNames.Authorization) != null && + !control.Public && !control.MustRevalidate && !control.SMaxAge.HasValue) + { + return false; + } + + return HeuristicallyCacheable.Contains(code) || control.Public || control.MaxAge.HasValue || + GetHeader(responseHeaders, HeaderNames.Expires) != null; + } + + public static Boolean HasValidator(HttpCacheEntry entry) => + GetHeader(entry.ResponseHeaders, HeaderNames.ETag) != null || + GetHeader(entry.ResponseHeaders, HeaderNames.LastModified) != null; + + public static Boolean IsConditional(Request request) => + GetHeader(request.Headers, HeaderNames.IfNoneMatch) != null || + GetHeader(request.Headers, HeaderNames.IfModifiedSince) != null || + GetHeader(request.Headers, HeaderNames.IfMatch) != null || + GetHeader(request.Headers, HeaderNames.IfUnmodifiedSince) != null || + GetHeader(request.Headers, "If-Range") != null; + + public static Boolean IsFreshEnough(HttpCacheEntry entry, CacheControl requestControl, DateTimeOffset now, Boolean heuristic) + { + var responseControl = CacheControl.Parse(entry.ResponseHeaders); + + if (responseControl.NoCache) + { + return false; + } + + var lifetime = GetFreshnessLifetime(entry, responseControl, heuristic); + var age = GetCurrentAge(entry, now); + + if (requestControl.MaxAge.HasValue && age > requestControl.MaxAge.Value) + { + return false; + } + + if (requestControl.MinFresh.HasValue) + { + return lifetime - age >= requestControl.MinFresh.Value; + } + + if (lifetime > age) + { + return true; + } + + if (responseControl.MustRevalidate) + { + return false; + } + + return requestControl.MaxStaleAny || + (requestControl.MaxStale.HasValue && age - lifetime <= requestControl.MaxStale.Value); + } + + public static Boolean MayServeStale(HttpCacheEntry entry) + { + var control = CacheControl.Parse(entry.ResponseHeaders); + return !control.MustRevalidate && !control.NoCache; + } + + public static TimeSpan GetFreshnessLifetime(HttpCacheEntry entry, CacheControl control, Boolean heuristic) + { + // s-maxage only applies to shared caches + if (control.MaxAge.HasValue) + { + return control.MaxAge.Value; + } + + var date = GetDate(entry); + var expires = GetHeader(entry.ResponseHeaders, HeaderNames.Expires); + + if (expires != null) + { + // invalid values (e.g., "0") represent a time in the past + return TryParseDate(expires, out var expiresDate) && expiresDate > date ? expiresDate - date : TimeSpan.Zero; + } + + var lastModified = GetHeader(entry.ResponseHeaders, HeaderNames.LastModified); + + if (heuristic && lastModified != null && TryParseDate(lastModified, out var modified) && modified < date && + (HeuristicallyCacheable.Contains((Int32)entry.StatusCode) || control.Public)) + { + var lifetime = TimeSpan.FromTicks((date - modified).Ticks / 10); + return lifetime < MaxHeuristicLifetime ? lifetime : MaxHeuristicLifetime; + } + + return TimeSpan.Zero; + } + + public static TimeSpan GetCurrentAge(HttpCacheEntry entry, DateTimeOffset now) + { + var date = GetDate(entry); + var ageValue = TimeSpan.Zero; + + if (Int64.TryParse(GetHeader(entry.ResponseHeaders, HeaderNames.Age), NumberStyles.None, CultureInfo.InvariantCulture, out var seconds)) + { + ageValue = TimeSpan.FromSeconds(Math.Min(seconds, Int32.MaxValue)); + } + + var apparentAge = Max(TimeSpan.Zero, entry.ResponseTime - date); + var responseDelay = entry.ResponseTime - entry.RequestTime; + var correctedAgeValue = ageValue + responseDelay; + var correctedInitialAge = Max(apparentAge, correctedAgeValue); + var residentTime = now - entry.ResponseTime; + return correctedInitialAge + residentTime; + } + + public static IEnumerable GetVary(IDictionary responseHeaders) + { + var vary = GetHeader(responseHeaders, HeaderNames.Vary); + + if (String.IsNullOrEmpty(vary)) + { + return Enumerable.Empty(); + } + + return vary.Split(',').Select(m => m.Trim()).Where(m => m.Length > 0); + } + + public static Boolean MatchesVary(HttpCacheEntry entry, Request request) + { + foreach (var name in GetVary(entry.ResponseHeaders)) + { + if (name == "*") + { + return false; + } + + var stored = Normalize(GetHeader(entry.RequestHeaders, name)); + var current = Normalize(GetHeader(request.Headers, name)); + + if (!String.Equals(stored, current, StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + + public static HttpCacheEntry CreateEntry(Request request, IResponse response, Byte[] body, DateTimeOffset requestTime, DateTimeOffset responseTime) + { + var entry = new HttpCacheEntry + { + Address = response.Address?.Href ?? request.Address.Href, + StatusCode = response.StatusCode, + RequestTime = requestTime, + ResponseTime = responseTime, + Body = body, + }; + + foreach (var header in response.Headers) + { + if (!UnstoredHeaders.Contains(header.Key, StringComparer.OrdinalIgnoreCase)) + { + entry.ResponseHeaders[header.Key] = header.Value; + } + } + + foreach (var name in GetVary(entry.ResponseHeaders)) + { + var value = GetHeader(request.Headers, name); + + if (value != null) + { + entry.RequestHeaders[name] = value; + } + } + + return entry; + } + + public static HttpCacheEntry Update(HttpCacheEntry stored, IDictionary notModifiedHeaders, DateTimeOffset requestTime, DateTimeOffset responseTime) + { + var entry = new HttpCacheEntry + { + Address = stored.Address, + StatusCode = stored.StatusCode, + RequestTime = requestTime, + ResponseTime = responseTime, + Body = stored.Body, + }; + + foreach (var header in stored.RequestHeaders) + { + entry.RequestHeaders[header.Key] = header.Value; + } + + foreach (var header in stored.ResponseHeaders) + { + entry.ResponseHeaders[header.Key] = header.Value; + } + + foreach (var header in notModifiedHeaders) + { + if (!UnmergedHeaders.Contains(header.Key, StringComparer.OrdinalIgnoreCase)) + { + entry.ResponseHeaders[header.Key] = header.Value; + } + } + + return entry; + } + + private static DateTimeOffset GetDate(HttpCacheEntry entry) + { + var value = GetHeader(entry.ResponseHeaders, HeaderNames.Date); + return value != null && TryParseDate(value, out var date) ? date : entry.ResponseTime; + } + + private static String Normalize(String value) + { + if (value == null) + { + return null; + } + + return String.Join(" ", value.Split(new[] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries)); + } + + private static TimeSpan Max(TimeSpan a, TimeSpan b) => a > b ? a : b; + } +} diff --git a/src/AngleSharp.Io/Http/IHttpCache.cs b/src/AngleSharp.Io/Http/IHttpCache.cs new file mode 100644 index 0000000..5eb191d --- /dev/null +++ b/src/AngleSharp.Io/Http/IHttpCache.cs @@ -0,0 +1,60 @@ +namespace AngleSharp.Io.Http +{ + using AngleSharp.Dom; + using System; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Represents the HTTP cache sitting in front of the HTTP(S) requesters. + /// + public interface IHttpCache + { + /// + /// Gets the options of the cache. + /// + HttpCacheOptions Options { get; } + + /// + /// Answers the request from the cache or by using the given network + /// callback, storing the response if permitted. + /// + /// The request to answer. + /// The callback for sending a request. + /// The token for cancelling the operation. + /// The task resolving to the response. + Task SendAsync(Request request, Func> network, CancellationToken cancel); + + /// + /// Looks up a stored response for the given request, regardless of + /// its freshness. + /// + /// The request to match. + /// The token for cancelling the operation. + /// The task resolving to the stored response, if any. + Task MatchAsync(Request request, CancellationToken cancel); + + /// + /// Removes all stored responses for the given address. + /// + /// The address to remove. + /// The token for cancelling the operation. + /// The task resolving to true if something was removed. + Task RemoveAsync(Url address, CancellationToken cancel); + + /// + /// Removes all stored responses of the given origin. + /// + /// The origin to clear, e.g., https://example.com. + /// The token for cancelling the operation. + /// The task completing when the origin has been cleared. + Task ClearAsync(String origin, CancellationToken cancel); + + /// + /// Removes all stored responses. + /// + /// The token for cancelling the operation. + /// The task completing when the cache has been cleared. + Task ClearAsync(CancellationToken cancel); + } +} diff --git a/src/AngleSharp.Io/Http/IHttpCacheHandler.cs b/src/AngleSharp.Io/Http/IHttpCacheHandler.cs new file mode 100644 index 0000000..d97a994 --- /dev/null +++ b/src/AngleSharp.Io/Http/IHttpCacheHandler.cs @@ -0,0 +1,48 @@ +namespace AngleSharp.Io.Http +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Represents a handler for reading and writing HTTP cache entries per + /// origin. Each key (the address without fragment) holds the variants + /// selected by the Vary header. + /// + public interface IHttpCacheHandler + { + /// + /// Reads the stored variants for the given key. + /// + /// The origin of the key. + /// The key to read. + /// The token for cancelling the operation. + /// The stored variants, which may be empty. + Task> ReadAsync(String origin, String key, CancellationToken cancel); + + /// + /// Writes the variants for the given key. Writing no variants removes + /// the key. + /// + /// The origin of the key. + /// The key to write. + /// The variants to store. + /// The token for cancelling the operation. + /// The task completing when the entries are written. + Task WriteAsync(String origin, String key, IEnumerable entries, CancellationToken cancel); + + /// + /// Gets the origins with stored entries. + /// + /// The known origins. + IEnumerable GetOrigins(); + + /// + /// Gets the stored keys of the given origin. + /// + /// The origin to look at. + /// The known keys. + IEnumerable GetKeys(String origin); + } +} diff --git a/src/AngleSharp.Io/Http/MemoryHttpCacheHandler.cs b/src/AngleSharp.Io/Http/MemoryHttpCacheHandler.cs new file mode 100644 index 0000000..d58ddc5 --- /dev/null +++ b/src/AngleSharp.Io/Http/MemoryHttpCacheHandler.cs @@ -0,0 +1,93 @@ +namespace AngleSharp.Io.Http +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Represents an in-memory HTTP cache handler. + /// + public class MemoryHttpCacheHandler : IHttpCacheHandler + { + private readonly Dictionary> _byOrigin; + + /// + /// Creates a new in-memory HTTP cache handler. + /// + public MemoryHttpCacheHandler() + { + _byOrigin = new Dictionary>(StringComparer.Ordinal); + } + + /// + public Task> ReadAsync(String origin, String key, CancellationToken cancel) + { + lock (_byOrigin) + { + if (_byOrigin.TryGetValue(origin ?? String.Empty, out var entries) && entries.TryGetValue(key, out var variants)) + { + return Task.FromResult>(variants); + } + } + + return Task.FromResult>(Array.Empty()); + } + + /// + public Task WriteAsync(String origin, String key, IEnumerable entries, CancellationToken cancel) + { + origin = origin ?? String.Empty; + var variants = entries?.ToArray() ?? Array.Empty(); + + lock (_byOrigin) + { + if (!_byOrigin.TryGetValue(origin, out var keys)) + { + if (variants.Length == 0) + { + return Task.CompletedTask; + } + + keys = new Dictionary(StringComparer.Ordinal); + _byOrigin[origin] = keys; + } + + if (variants.Length > 0) + { + keys[key] = variants; + } + else if (keys.Remove(key) && keys.Count == 0) + { + _byOrigin.Remove(origin); + } + } + + return Task.CompletedTask; + } + + /// + public IEnumerable GetOrigins() + { + lock (_byOrigin) + { + return _byOrigin.Keys.ToArray(); + } + } + + /// + public IEnumerable GetKeys(String origin) + { + lock (_byOrigin) + { + if (_byOrigin.TryGetValue(origin ?? String.Empty, out var keys)) + { + return keys.Keys.ToArray(); + } + } + + return Array.Empty(); + } + } +} diff --git a/src/AngleSharp.Io/Http/PersistenceHttpCacheHandler.cs b/src/AngleSharp.Io/Http/PersistenceHttpCacheHandler.cs new file mode 100644 index 0000000..754e354 --- /dev/null +++ b/src/AngleSharp.Io/Http/PersistenceHttpCacheHandler.cs @@ -0,0 +1,266 @@ +namespace AngleSharp.Io.Http +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Security.Cryptography; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Represents a persistent HTTP cache handler writing one directory per + /// origin and one file per key. + /// + public class PersistenceHttpCacheHandler : IHttpCacheHandler + { + private const Int32 FormatVersion = 1; + private const String OriginFileName = "origin.txt"; + private const String EntryExtension = ".entry"; + + private readonly String _directoryPath; + + /// + /// Creates a new persistent HTTP cache handler for the given directory. + /// + /// The directory used to store the entries. + public PersistenceHttpCacheHandler(String directoryPath) + { + _directoryPath = directoryPath ?? throw new ArgumentNullException(nameof(directoryPath)); + Directory.CreateDirectory(_directoryPath); + } + + /// + public async Task> ReadAsync(String origin, String key, CancellationToken cancel) + { + var filePath = GetEntryPath(origin, key); + + try + { + if (File.Exists(filePath)) + { + var content = await ReadAllBytesAsync(filePath, cancel).ConfigureAwait(false); + return Deserialize(content, key); + } + } + catch (IOException) + { + // a concurrent write or a damaged (truncated) file is a cache miss + } + catch (ArgumentException) + { + // a damaged file may also yield negative lengths + } + + return Array.Empty(); + } + + /// + public async Task WriteAsync(String origin, String key, IEnumerable entries, CancellationToken cancel) + { + var originPath = GetOriginPath(origin); + var filePath = GetEntryPath(origin, key); + var variants = entries?.ToArray() ?? Array.Empty(); + + if (variants.Length == 0) + { + if (File.Exists(filePath)) + { + File.Delete(filePath); + } + + if (Directory.Exists(originPath) && !Directory.EnumerateFiles(originPath, "*" + EntryExtension).Any()) + { + Directory.Delete(originPath, true); + } + + return; + } + + Directory.CreateDirectory(originPath); + File.WriteAllText(Path.Combine(originPath, OriginFileName), origin ?? String.Empty); + + // write next to the target and swap afterwards, such that readers + // never observe a half-written entry + var content = Serialize(key, variants); + var tempPath = filePath + "." + Guid.NewGuid().ToString("N") + ".tmp"; + + using (var stream = new FileStream(tempPath, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true)) + { + await stream.WriteAsync(content, 0, content.Length, cancel).ConfigureAwait(false); + } + + if (File.Exists(filePath)) + { + File.Replace(tempPath, filePath, null); + } + else + { + File.Move(tempPath, filePath); + } + } + + /// + public IEnumerable GetOrigins() + { + var result = new List(); + + foreach (var directory in Directory.EnumerateDirectories(_directoryPath)) + { + var originFile = Path.Combine(directory, OriginFileName); + + if (File.Exists(originFile)) + { + result.Add(File.ReadAllText(originFile)); + } + } + + return result; + } + + /// + public IEnumerable GetKeys(String origin) + { + var originPath = GetOriginPath(origin); + var result = new List(); + + if (Directory.Exists(originPath)) + { + foreach (var file in Directory.EnumerateFiles(originPath, "*" + EntryExtension)) + { + try + { + using (var reader = new BinaryReader(File.OpenRead(file), Encoding.UTF8)) + { + if (reader.ReadInt32() == FormatVersion) + { + result.Add(reader.ReadString()); + } + } + } + catch (IOException) + { + } + } + } + + return result; + } + + private String GetOriginPath(String origin) => + Path.Combine(_directoryPath, Hash(origin ?? String.Empty)); + + private String GetEntryPath(String origin, String key) => + Path.Combine(GetOriginPath(origin), Hash(key ?? String.Empty) + EntryExtension); + + private static async Task ReadAllBytesAsync(String filePath, CancellationToken cancel) + { + using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) + using (var memory = new MemoryStream()) + { + await stream.CopyToAsync(memory, 81920, cancel).ConfigureAwait(false); + return memory.ToArray(); + } + } + + private static Byte[] Serialize(String key, HttpCacheEntry[] variants) + { + using (var memory = new MemoryStream()) + { + using (var writer = new BinaryWriter(memory, Encoding.UTF8)) + { + writer.Write(FormatVersion); + writer.Write(key ?? String.Empty); + writer.Write(variants.Length); + + foreach (var entry in variants) + { + writer.Write(entry.Address ?? String.Empty); + writer.Write((Int32)entry.StatusCode); + writer.Write(entry.RequestTime.UtcTicks); + writer.Write(entry.ResponseTime.UtcTicks); + WriteHeaders(writer, entry.RequestHeaders); + WriteHeaders(writer, entry.ResponseHeaders); + var body = entry.Body ?? Array.Empty(); + writer.Write(body.Length); + writer.Write(body); + } + } + + return memory.ToArray(); + } + } + + private static IReadOnlyList Deserialize(Byte[] content, String key) + { + using (var reader = new BinaryReader(new MemoryStream(content), Encoding.UTF8)) + { + // a file of another format version or a hash collision is a miss + if (reader.ReadInt32() != FormatVersion || reader.ReadString() != key) + { + return Array.Empty(); + } + + var count = reader.ReadInt32(); + var result = new List(count); + + for (var i = 0; i < count; i++) + { + var entry = new HttpCacheEntry + { + Address = reader.ReadString(), + StatusCode = (HttpStatusCode)reader.ReadInt32(), + RequestTime = new DateTimeOffset(reader.ReadInt64(), TimeSpan.Zero), + ResponseTime = new DateTimeOffset(reader.ReadInt64(), TimeSpan.Zero), + }; + ReadHeaders(reader, entry.RequestHeaders); + ReadHeaders(reader, entry.ResponseHeaders); + entry.Body = reader.ReadBytes(reader.ReadInt32()); + result.Add(entry); + } + + return result; + } + } + + private static void WriteHeaders(BinaryWriter writer, IDictionary headers) + { + writer.Write(headers.Count); + + foreach (var header in headers) + { + writer.Write(header.Key); + writer.Write(header.Value ?? String.Empty); + } + } + + private static void ReadHeaders(BinaryReader reader, IDictionary headers) + { + var count = reader.ReadInt32(); + + for (var i = 0; i < count; i++) + { + var name = reader.ReadString(); + headers[name] = reader.ReadString(); + } + } + + private static String Hash(String value) + { + using (var sha = SHA256.Create()) + { + var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(value)); + var sb = new StringBuilder(hash.Length * 2); + + for (var i = 0; i < hash.Length; i++) + { + sb.Append(hash[i].ToString("x2")); + } + + return sb.ToString(); + } + } + } +} diff --git a/src/AngleSharp.Io/Http/PrefixedStream.cs b/src/AngleSharp.Io/Http/PrefixedStream.cs new file mode 100644 index 0000000..4e6e786 --- /dev/null +++ b/src/AngleSharp.Io/Http/PrefixedStream.cs @@ -0,0 +1,86 @@ +namespace AngleSharp.Io.Http +{ + using System; + using System.IO; + using System.Threading; + using System.Threading.Tasks; + + /// + /// A read-only stream replaying already consumed bytes before continuing + /// with the remainder of the original stream. + /// + internal sealed class PrefixedStream : Stream + { + private readonly Byte[] _prefix; + private readonly Stream _remainder; + private Int32 _position; + + public PrefixedStream(Byte[] prefix, Stream remainder) + { + _prefix = prefix; + _remainder = remainder; + } + + public override Boolean CanRead => true; + + public override Boolean CanSeek => false; + + public override Boolean CanWrite => false; + + public override Int64 Length => throw new NotSupportedException(); + + public override Int64 Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override Int32 Read(Byte[] buffer, Int32 offset, Int32 count) + { + if (_position < _prefix.Length) + { + return ReadPrefix(buffer, offset, count); + } + + return _remainder.Read(buffer, offset, count); + } + + public override Task ReadAsync(Byte[] buffer, Int32 offset, Int32 count, CancellationToken cancellationToken) + { + if (_position < _prefix.Length) + { + return Task.FromResult(ReadPrefix(buffer, offset, count)); + } + + return _remainder.ReadAsync(buffer, offset, count, cancellationToken); + } + + public override void Flush() + { + } + + public override Int64 Seek(Int64 offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(Int64 value) => throw new NotSupportedException(); + + public override void Write(Byte[] buffer, Int32 offset, Int32 count) => throw new NotSupportedException(); + + protected override void Dispose(Boolean disposing) + { + if (disposing) + { + _remainder.Dispose(); + } + + base.Dispose(disposing); + } + + private Int32 ReadPrefix(Byte[] buffer, Int32 offset, Int32 count) + { + var length = Math.Min(count, _prefix.Length - _position); + Buffer.BlockCopy(_prefix, _position, buffer, offset, length); + _position += length; + return length; + } + } +} diff --git a/src/AngleSharp.Io/IoConfigurationExtensions.cs b/src/AngleSharp.Io/IoConfigurationExtensions.cs index a7a4c43..c6b7128 100644 --- a/src/AngleSharp.Io/IoConfigurationExtensions.cs +++ b/src/AngleSharp.Io/IoConfigurationExtensions.cs @@ -6,6 +6,7 @@ namespace AngleSharp; using AngleSharp.Io.Dom; using AngleSharp.Io.IndexedDb; using AngleSharp.Io.GeolocationApi; +using AngleSharp.Io.Http; using AngleSharp.Io.Network; using AngleSharp.Io.Storage; using AngleSharp.Io.ClipboardApi; @@ -39,6 +40,7 @@ public static IConfiguration WithIo(this IConfiguration configuration, IoOptions .WithNavigator(options.Navigator) .WithCookies(options.CookieHandler) .WithRequesters(options.HttpHandler) + .WithHttpCache(options.HttpCache) .WithStorageProviderFactory(options.StorageFactory) .WithIndexedDbProviderFactory(options.IndexedDbFactory) .WithCacheProviderFactory(options.CacheFactory) @@ -190,6 +192,85 @@ public static IConfiguration WithRequester(this IConfiguration configuration) #endregion + #region HTTP Cache + + /// + /// Wraps the registered HTTP(S) requesters with an HTTP cache keeping + /// its entries in memory. Needs to be called after the requesters have + /// been registered. + /// + /// The configuration to extend. + /// The new configuration. + public static IConfiguration WithHttpCache(this IConfiguration configuration) => + configuration.WithHttpCache(new HttpCacheOptions()); + + /// + /// Wraps the registered HTTP(S) requesters with an HTTP cache persisting + /// its entries in the given directory. Needs to be called after the + /// requesters have been registered. + /// + /// The configuration to extend. + /// The directory for storing the entries. + /// The new configuration. + public static IConfiguration WithHttpCache(this IConfiguration configuration, String directoryPath) => + configuration.WithHttpCache(new PersistenceHttpCacheHandler(directoryPath)); + + /// + /// Wraps the registered HTTP(S) requesters with an HTTP cache using the + /// given handler. Needs to be called after the requesters have been + /// registered. + /// + /// The configuration to extend. + /// The handler for storing the entries. + /// The new configuration. + public static IConfiguration WithHttpCache(this IConfiguration configuration, IHttpCacheHandler handler) => + configuration.WithHttpCache(new HttpCacheOptions { Handler = handler }); + + /// + /// Wraps the registered HTTP(S) requesters with an HTTP cache using the + /// given options. Needs to be called after the requesters have been + /// registered. Passing null leaves the configuration unchanged. + /// + /// The configuration to extend. + /// The options of the cache. + /// The new configuration. + public static IConfiguration WithHttpCache(this IConfiguration configuration, HttpCacheOptions options) + { + if (options is null) + { + return configuration; + } + + return configuration.WithHttpCache(new HttpCache(options)); + } + + /// + /// Wraps the registered HTTP(S) requesters with the given HTTP cache, + /// which is also registered as a service. Needs to be called after the + /// requesters have been registered. + /// + /// The configuration to extend. + /// The cache to use. + /// The new configuration. + public static IConfiguration WithHttpCache(this IConfiguration configuration, IHttpCache cache) + { + // rebuilding the list keeps the order of the requesters, which + // decides which requester handles a protocol + var services = configuration.Services.Select(service => service switch + { + CachingRequester caching => new CachingRequester(caching.Inner, cache), + IRequester requester when IsHttpRequester(requester) => new CachingRequester(requester, cache), + _ => service, + }); + + return new Configuration(services).WithOnly(cache); + } + + private static Boolean IsHttpRequester(IRequester requester) => + requester.SupportsProtocol(ProtocolNames.Http) || requester.SupportsProtocol(ProtocolNames.Https); + + #endregion + #region Cookies /// diff --git a/src/AngleSharp.Io/IoOptions.cs b/src/AngleSharp.Io/IoOptions.cs index ce4017f..1e8f7a5 100644 --- a/src/AngleSharp.Io/IoOptions.cs +++ b/src/AngleSharp.Io/IoOptions.cs @@ -1,9 +1,12 @@ namespace AngleSharp.Io; +using System; +using System.IO; using System.Net.Http; using AngleSharp.Io.Cache; using AngleSharp.Io.Cookie; using AngleSharp.Io.Dom; +using AngleSharp.Io.Http; using AngleSharp.Io.IndexedDb; using AngleSharp.Io.Storage; @@ -12,6 +15,42 @@ namespace AngleSharp.Io; /// public sealed class IoOptions { + /// + /// Creates options keeping all data (cookies, storage, HTTP cache) in + /// memory only. + /// + /// The new options. + public static IoOptions CreateTemporary() => new IoOptions + { + HttpCache = new HttpCacheOptions(), + }; + + /// + /// Creates options persisting the cookies (cookies.txt), the local storage + /// (storage) and the HTTP cache (http-cache) below the given directory. + /// + /// The directory holding the data. + /// The new options. + public static IoOptions CreatePersistent(String rootDirectoryPath) + { + if (rootDirectoryPath is null) + { + throw new ArgumentNullException(nameof(rootDirectoryPath)); + } + + Directory.CreateDirectory(rootDirectoryPath); + + return new IoOptions + { + CookieHandler = new LocalFileHandler(Path.Combine(rootDirectoryPath, "cookies.txt")), + StorageFactory = StorageProviderFactory.CreatePersistent(Path.Combine(rootDirectoryPath, "storage")), + HttpCache = new HttpCacheOptions + { + Handler = new PersistenceHttpCacheHandler(Path.Combine(rootDirectoryPath, "http-cache")), + }, + }; + } + /// /// Gets or sets the options for configuring the navigator. /// @@ -27,6 +66,13 @@ public sealed class IoOptions /// public HttpClientHandler HttpHandler { get; set; } = new HttpClientHandler(); + /// + /// Gets or sets the options for configuring the HTTP cache in front of the + /// HTTP requester. By default, this is null, which means that responses + /// are not cached. + /// + public HttpCacheOptions HttpCache { get; set; } = null; + /// /// Gets or sets the provider for configuring the web storage provider. /// From 0f4f2de5e60a41a4d95e1a8321c0cd13254ed790 Mon Sep 17 00:00:00 2001 From: Florian Rappl Date: Mon, 19 Oct 2026 22:50:28 +0200 Subject: [PATCH 05/10] Updated changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59aa91f..3fd74ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # 1.2.0 -Released on tbd. +Released on Wednesday, October 21 2026.. - Added an HTTP cache for the requesters (`WithHttpCache`) - Added `IoOptions.CreateTemporary` and `IoOptions.CreatePersistent` From f69e700964fab5b2a8ba546fa00812e4fb885174 Mon Sep 17 00:00:00 2001 From: Florian Rappl Date: Mon, 19 Oct 2026 22:53:39 +0200 Subject: [PATCH 06/10] Document the HTTP cache Covers WithHttpCache, the caching rules, HttpCacheOptions and modes, IHttpCache, custom handlers, IoOptions.CreateTemporary/CreatePersistent, an offline replay workflow, and FAQ entries. Claude-Session: https://claude.ai/code/session_01Je6aX4Be36K1jC6MTpDng4 --- docs/general/01-Basics.md | 109 +++++++++++++++++++++++++- docs/tutorials/01-API.md | 14 ++++ docs/tutorials/03-Common-Workflows.md | 44 +++++++++++ docs/tutorials/04-Questions.md | 26 ++++++ 4 files changed, 192 insertions(+), 1 deletion(-) diff --git a/docs/general/01-Basics.md b/docs/general/01-Basics.md index 7238962..4700a73 100644 --- a/docs/general/01-Basics.md +++ b/docs/general/01-Basics.md @@ -50,6 +50,25 @@ var config = Configuration.Default `WithIo` wires navigator, cookies, requesters, storage, IndexedDB, and cache in one call. Clipboard and geolocation are only enabled when platforms are provided. +Instead of `new IoOptions()` you can start from one of the factory methods, which also turn on +the [HTTP cache](#http-cache): + +```cs +// everything in memory, e.g., for tests or private sessions +var temporary = IoOptions.CreateTemporary(); + +// cookies, local storage, and HTTP cache persisted below one directory: +// /cookies.txt, /storage, /http-cache +var persistent = IoOptions.CreatePersistent(rootDirectory); + +var config = Configuration.Default + .WithIo(persistent) + .WithDefaultLoader(); +``` + +A plain `new IoOptions()` leaves the HTTP cache off. IndexedDB and Cache Storage are always kept in +memory, also with `CreatePersistent`. + ### Requesters If you just want to use *all* available requesters provided by AngleSharp.Io you can do the following: @@ -93,6 +112,92 @@ var config = Configuration.Default .WithDefaultLoader(); ``` +### HTTP Cache + +The HTTP(S) requesters can be wrapped with a private HTTP cache that follows RFC 9111, just like +the cache of a browser. Cached responses are answered without touching the network; stale ones are +revalidated with the server. + +```cs +var config = Configuration.Default + .WithRequesters() // the cache wraps the requesters registered so far ... + .WithHttpCache() // ... so it needs to come after them + .WithDefaultLoader(); +``` + +The entries can be kept in memory (default) or persisted in a directory, which is the same choice +as for cookies and storage: + +```cs +config.WithHttpCache(); // memory +config.WithHttpCache(cacheDirectory); // persisted files +config.WithHttpCache(new MemoryHttpCacheHandler()); // explicit handler +config.WithHttpCache(new HttpCacheOptions { /* see below */ }); // full control +``` + +What gets cached: + +- Only `GET` responses are stored; a `HEAD` request is answered from a stored `GET` response. +- Unsafe methods (`POST`, `PUT`, `DELETE`, ...) are never cached. A successful one removes the stored + responses for its address as well as for same-origin `Location` / `Content-Location` targets. +- `Cache-Control` (`no-store`, `no-cache`, `max-age`, `must-revalidate`, `public`, ...) and `Expires` + of the response decide how long a response is fresh. Without them, responses with `Last-Modified` + stay fresh for 10% of their age (at most 7 days). +- Request headers are respected, too: `Cache-Control: no-store`, `no-cache`, `max-age`, `min-fresh`, + `max-stale`, `only-if-cached`, and `Pragma: no-cache`. +- `Vary` keeps separate variants, e.g., per `Accept-Language`. `Vary: *` is never stored. +- Stale responses with `ETag` or `Last-Modified` are revalidated using `If-None-Match` / + `If-Modified-Since`. A `304` refreshes the stored response. +- Requests with an `Authorization` header are only stored if the response allows it (`public`, + `must-revalidate`, or `s-maxage`). `private` responses are stored, since the cache is private. +- `Set-Cookie` is never stored, so cookies are only applied for responses coming from the network. +- If the network fails, a stale response is used instead, unless it requires revalidation. + +The `HttpCacheOptions` give you more control: + +```cs +var options = new HttpCacheOptions +{ + Handler = new PersistenceHttpCacheHandler(cacheDirectory), + Mode = HttpCacheMode.Default, + MaxEntrySize = 10 * 1024 * 1024, // larger responses are passed through, but not stored + UseHeuristicFreshness = true, // the Last-Modified rule from above + Bypass = request => request.Address.Path.StartsWith("api/"), // never cache these +}; + +var config = Configuration.Default + .WithRequesters() + .WithHttpCache(options) + .WithDefaultLoader(); +``` + +The `Mode` uses the names of the `cache` option of `fetch`: + +| Mode | Uses stored responses | Goes to the network | Stores responses | +| --- | --- | --- | --- | +| `Default` | if fresh (stale ones are revalidated) | if needed | yes | +| `NoStore` | no | always | no | +| `Reload` | no | always | yes | +| `NoCache` | after revalidation | always | yes | +| `ForceCache` | always, even if stale | only if nothing is stored | yes | +| `OnlyIfCached` | always, even if stale | never (`504` if nothing is stored) | no | + +The cache is registered as `IHttpCache`, which can be used to look into or clear the entries. The +entries are kept per origin of the requested address: + +```cs +var context = BrowsingContext.New(config); +var cache = context.GetService(); + +var stored = await cache.MatchAsync(new Request { Address = new Url("https://example.org/") }, CancellationToken.None); +await cache.RemoveAsync(new Url("https://example.org/"), CancellationToken.None); +await cache.ClearAsync("https://example.org", CancellationToken.None); // one origin +await cache.ClearAsync(CancellationToken.None); // everything +``` + +Note that the HTTP cache is not the same as the [Cache Storage](#cache-storage) (`caches`), which is +a DOM API that is only filled explicitly. + ### Cookies To get improved cookie support, e.g., do @@ -233,7 +338,9 @@ using (var writeTx = upgradedAgain.BeginTransaction("records", IndexedDbTransact ### Cache Storage -AngleSharp.Io also exposes a lightweight Cache Storage surface that is shared per origin: +AngleSharp.Io also exposes a lightweight Cache Storage surface that is shared per origin. This is +the DOM API (`caches`) that scripts fill explicitly; for caching the responses of the requesters +see [HTTP Cache](#http-cache). - `caches` using in-memory caches and response snapshots scoped to the origin diff --git a/docs/tutorials/01-API.md b/docs/tutorials/01-API.md index 7f885a6..3b2b7ac 100644 --- a/docs/tutorials/01-API.md +++ b/docs/tutorials/01-API.md @@ -19,3 +19,17 @@ The content could be either a byte array or `Stream`. Most importantly, the `AdvancedCookieProvider` is a much better way of handling cookies. While AngleSharp itself uses the cookie jar provided by .NET, AngleSharp.Io has reimplemented a cookie provider following the official specification. This way, the cookie handling just works. Cookies can be handled temporarily (`MemoryFileHandler`) or persistently (`LocalFileHandler`). Custom file handlers for actually storing or retrieving the cookie content are possible. The crucial interface for this is `ICookieFileHandler`. + +The HTTP cache (`HttpCache`, registered as `IHttpCache`) sits in front of the HTTP(S) requesters by wrapping them in a `CachingRequester`. The `CachingRequester` can also be used directly to wrap any other `IRequester`: + +```cs +var cache = new HttpCache(new HttpCacheOptions()); +var requester = new CachingRequester(new HttpClientRequester(myHttpClient), cache); + +var config = Configuration.Default + .With(requester) + .WithOnly(cache) + .WithDefaultLoader(); +``` + +Like the cookies, the cache entries can be stored temporarily (`MemoryHttpCacheHandler`) or persistently (`PersistenceHttpCacheHandler`). Custom handlers, e.g., for storing the entries in a database, implement `IHttpCacheHandler`. A handler reads and writes the variants (`HttpCacheEntry`) of a key (the address without fragment) within an origin; writing no variants removes the key. diff --git a/docs/tutorials/03-Common-Workflows.md b/docs/tutorials/03-Common-Workflows.md index a54b0d6..e7fdb97 100644 --- a/docs/tutorials/03-Common-Workflows.md +++ b/docs/tutorials/03-Common-Workflows.md @@ -27,6 +27,10 @@ Why this helps: - You configure navigator, cookies, requesters, storage, IndexedDB, and cache together. - You can still override any dependency through `IoOptions`. +To keep a whole browsing profile (cookies, local storage, HTTP cache) on disk, start from +`IoOptions.CreatePersistent(rootDirectory)` instead; `IoOptions.CreateTemporary()` does the same in +memory. + ## 2. Use custom HttpClient behavior with requesters Use this when you need proxying, custom decompression, headers, or controlled redirect behavior. @@ -135,3 +139,43 @@ await closed.Task; ``` Tip: for tests that depend on external endpoints, use explicit timeouts to avoid indefinite waits. + +## 7. Cache responses and replay them offline + +Use this for crawlers that revisit pages, or for tests that should run against previously recorded +responses instead of the live site. + +```cs +var cacheDirectory = Path.Combine(Path.GetTempPath(), "anglesharp.http-cache"); + +// first run: fill the cache while browsing as usual +var recording = Configuration.Default + .WithRequesters() + .WithHttpCache(new HttpCacheOptions + { + Handler = new PersistenceHttpCacheHandler(cacheDirectory), + Mode = HttpCacheMode.ForceCache, // keep using what is stored, even if stale + }) + .WithDefaultLoader(); + +await BrowsingContext.New(recording).OpenAsync("https://example.org"); + +// later runs: never touch the network +var replaying = Configuration.Default + .WithRequesters() + .WithHttpCache(new HttpCacheOptions + { + Handler = new PersistenceHttpCacheHandler(cacheDirectory), + Mode = HttpCacheMode.OnlyIfCached, // 504 for anything not recorded + }) + .WithDefaultLoader(); + +var document = await BrowsingContext.New(replaying).OpenAsync("https://example.org"); +``` + +Notes: + +- Only responses the server allows to be cached are recorded, e.g., `no-store` responses and `Vary: *` + are never stored. +- With `HttpCacheMode.Default` the cache behaves like a browser and only uses fresh responses. + diff --git a/docs/tutorials/04-Questions.md b/docs/tutorials/04-Questions.md index aa189f3..0b98ce8 100644 --- a/docs/tutorials/04-Questions.md +++ b/docs/tutorials/04-Questions.md @@ -35,3 +35,29 @@ var config = Configuration.Default .With(new HttpClientRequester()) // only requester .WithDefaultLoader(); ``` + +## How can responses be cached? + +Use the `WithHttpCache` extension method after the requesters have been registered: + +```cs +var config = Configuration.Default + .WithRequesters() // from AngleSharp.Io + .WithHttpCache() // wraps the HTTP(S) requesters registered above + .WithDefaultLoader(); +``` + +Again, the order matters: `WithHttpCache` only wraps the requesters that are already registered. Pass a directory (`WithHttpCache(cacheDirectory)`) to keep the cache across runs. + +## Why is a response not taken from the cache? + +The HTTP cache follows the rules of a browser cache (RFC 9111). Common reasons for a response not being served from the cache are: + +- the request is not a `GET` (or `HEAD`) request +- the response has `Cache-Control: no-store` or `Vary: *` +- the response has no `Cache-Control: max-age`, `Expires`, or `Last-Modified`, so it is stale immediately and revalidated (or fetched again) on the next request +- the request had an `Authorization` header and the response is not `public` +- the response is larger than `HttpCacheOptions.MaxEntrySize` + +To use stored responses regardless of their freshness, set `Mode = HttpCacheMode.ForceCache` (or `OnlyIfCached` to never use the network) in the `HttpCacheOptions`. + From bf2a6bf6f064c86d2fcded7f2e6100098409dd18 Mon Sep 17 00:00:00 2001 From: Florian Rappl Date: Mon, 19 Oct 2026 22:55:58 +0200 Subject: [PATCH 07/10] Prepare new version --- CHANGELOG.md | 2 +- src/AngleSharp.Io.Docs/package.json | 2 +- src/Directory.Build.props | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fd74ac..bbe3bf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # 1.2.0 -Released on Wednesday, October 21 2026.. +Released on Wednesday, October 21 2026. - Added an HTTP cache for the requesters (`WithHttpCache`) - Added `IoOptions.CreateTemporary` and `IoOptions.CreatePersistent` diff --git a/src/AngleSharp.Io.Docs/package.json b/src/AngleSharp.Io.Docs/package.json index e0eed31..8c0186d 100644 --- a/src/AngleSharp.Io.Docs/package.json +++ b/src/AngleSharp.Io.Docs/package.json @@ -1,6 +1,6 @@ { "name": "@anglesharp/io", - "version": "1.1.0", + "version": "1.2.0", "preview": true, "description": "The doclet for the AngleSharp.Io documentation.", "keywords": [ diff --git a/src/Directory.Build.props b/src/Directory.Build.props index e7d70ce..d68e910 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -2,7 +2,7 @@ Providers additional requesters and IO helpers for AngleSharp. AngleSharp.Io - 1.1.0 + 1.2.0 latest true true From 49ec3c9285b11e9341c1c6314f32510593c2d130 Mon Sep 17 00:00:00 2001 From: Florian Rappl Date: Tue, 20 Oct 2026 10:53:21 +0200 Subject: [PATCH 08/10] Implemented more APIs --- AGENTS.md | 4 + CHANGELOG.md | 6 + README.md | 331 +++++- docs/general/01-Basics.md | 325 +++++- docs/tutorials/03-Common-Workflows.md | 3 + src/AngleSharp.Io.Tests/Beacon/BeaconTests.cs | 655 ++++++++++++ .../IoConfigurationExtensionsTests.cs | 15 + .../Network/XmlHttpRequesterTests.cs | 24 +- .../Permissions/PermissionsTests.cs | 354 +++++++ .../Reporting/ReportingTests.cs | 800 ++++++++++++++ .../ServiceWorker/ServiceWorkerTests.cs | 815 +++++++++++++++ .../StorageManager/StorageManagerTests.cs | 286 ++++++ .../WebRtc/RtcPeerConnectionTests.cs | 972 ++++++++++++++++++ src/AngleSharp.Io/BeaconOptions.cs | 29 + .../Clipboard/ClipboardProviderFactory.cs | 2 +- src/AngleSharp.Io/Dom/BeaconSender.cs | 315 ++++++ src/AngleSharp.Io/Dom/Clipboard.cs | 32 +- src/AngleSharp.Io/Dom/Geolocation.cs | 19 + src/AngleSharp.Io/Dom/LockManager.cs | 2 +- src/AngleSharp.Io/Dom/NavigatorExtensions.cs | 85 +- .../Dom/PermissionChangedEventArgs.cs | 30 + src/AngleSharp.Io/Dom/PermissionDescriptor.cs | 17 + src/AngleSharp.Io/Dom/PermissionState.cs | 27 + src/AngleSharp.Io/Dom/PermissionStatus.cs | 101 ++ src/AngleSharp.Io/Dom/Permissions.cs | 105 ++ src/AngleSharp.Io/Dom/ReportingObserver.cs | 225 ++++ src/AngleSharp.Io/Dom/RequestBody.cs | 79 ++ src/AngleSharp.Io/Dom/ServiceWorker.cs | 89 ++ .../Dom/ServiceWorkerContainer.cs | 272 +++++ .../Dom/ServiceWorkerRegistration.cs | 86 ++ src/AngleSharp.Io/Dom/ServiceWorkerState.cs | 42 + .../Dom/ServiceWorkerUpdateViaCache.cs | 27 + src/AngleSharp.Io/Dom/StorageManager.cs | 118 +++ src/AngleSharp.Io/Dom/WindowExtensions.cs | 102 +- src/AngleSharp.Io/Dom/WorkerType.cs | 22 + src/AngleSharp.Io/Dom/XmlHttpRequest.cs | 66 +- .../Geolocation/GeolocationProviderFactory.cs | 2 +- .../Interfaces/IPermissionPolicy.cs | 41 + .../Interfaces/IServiceWorkerPlatform.cs | 58 ++ .../IServiceWorkerProviderFactory.cs | 17 + .../Interfaces/IStorageManagerPlatform.cs | 32 + .../IStorageManagerProviderFactory.cs | 17 + .../IoConfigurationExtensions.cs | 258 ++++- src/AngleSharp.Io/IoOptions.cs | 42 + .../Network/HttpClientRequester.cs | 11 +- .../Permissions/PermissionNames.cs | 43 + .../Permissions/PermissionPolicy.cs | 226 ++++ .../Reporting/EndpointReportSink.cs | 75 ++ src/AngleSharp.Io/Reporting/IReportSink.cs | 21 + .../Reporting/MemoryReportSink.cs | 50 + .../Reporting/NetworkErrorReportBody.cs | 75 ++ .../Reporting/NetworkFailures.cs | 26 + src/AngleSharp.Io/Reporting/Report.cs | 108 ++ src/AngleSharp.Io/Reporting/ReportBody.cs | 54 + .../Reporting/ReportEventArgs.cs | 24 + src/AngleSharp.Io/Reporting/ReportFilter.cs | 56 + src/AngleSharp.Io/Reporting/ReportJson.cs | 166 +++ src/AngleSharp.Io/Reporting/ReportTypes.cs | 34 + .../Reporting/ReportingOptions.cs | 125 +++ .../Reporting/ReportingRequester.cs | 199 ++++ .../Reporting/ReportingService.cs | 228 ++++ .../ServiceWorker/ServiceWorkerPlatform.cs | 59 ++ .../ServiceWorkerProviderFactory.cs | 46 + .../ServiceWorker/ServiceWorkerRegistry.cs | 390 +++++++ .../ServiceWorker/ServiceWorkerRequester.cs | 88 ++ .../MemoryStorageManagerPlatform.cs | 73 ++ .../StorageManagerProviderFactory.cs | 45 + src/AngleSharp.Io/WebRtc/IRtcPlatform.cs | 205 ++++ .../WebRtc/LoopbackRtcPlatform.cs | 533 ++++++++++ src/AngleSharp.Io/WebRtc/RtcBundlePolicy.cs | 27 + src/AngleSharp.Io/WebRtc/RtcConfiguration.cs | 138 +++ src/AngleSharp.Io/WebRtc/RtcDataChannel.cs | 326 ++++++ .../WebRtc/RtcDataChannelState.cs | 32 + src/AngleSharp.Io/WebRtc/RtcEvents.cs | 80 ++ src/AngleSharp.Io/WebRtc/RtcIceCandidate.cs | 223 ++++ .../WebRtc/RtcIceCandidateType.cs | 32 + src/AngleSharp.Io/WebRtc/RtcIceComponent.cs | 22 + .../WebRtc/RtcIceConnectionState.cs | 47 + .../WebRtc/RtcIceGatheringState.cs | 27 + src/AngleSharp.Io/WebRtc/RtcIceProtocol.cs | 22 + .../WebRtc/RtcIceTcpCandidateType.cs | 27 + .../WebRtc/RtcIceTransportPolicy.cs | 22 + src/AngleSharp.Io/WebRtc/RtcPeerConnection.cs | 943 +++++++++++++++++ .../WebRtc/RtcPeerConnectionState.cs | 42 + src/AngleSharp.Io/WebRtc/RtcRtcpMuxPolicy.cs | 17 + src/AngleSharp.Io/WebRtc/RtcSdpType.cs | 32 + .../WebRtc/RtcSessionDescription.cs | 78 ++ src/AngleSharp.Io/WebRtc/RtcSignalingState.cs | 42 + 88 files changed, 11926 insertions(+), 162 deletions(-) create mode 100644 src/AngleSharp.Io.Tests/Beacon/BeaconTests.cs create mode 100644 src/AngleSharp.Io.Tests/Permissions/PermissionsTests.cs create mode 100644 src/AngleSharp.Io.Tests/Reporting/ReportingTests.cs create mode 100644 src/AngleSharp.Io.Tests/ServiceWorker/ServiceWorkerTests.cs create mode 100644 src/AngleSharp.Io.Tests/StorageManager/StorageManagerTests.cs create mode 100644 src/AngleSharp.Io.Tests/WebRtc/RtcPeerConnectionTests.cs create mode 100644 src/AngleSharp.Io/BeaconOptions.cs create mode 100644 src/AngleSharp.Io/Dom/BeaconSender.cs create mode 100644 src/AngleSharp.Io/Dom/PermissionChangedEventArgs.cs create mode 100644 src/AngleSharp.Io/Dom/PermissionDescriptor.cs create mode 100644 src/AngleSharp.Io/Dom/PermissionState.cs create mode 100644 src/AngleSharp.Io/Dom/PermissionStatus.cs create mode 100644 src/AngleSharp.Io/Dom/Permissions.cs create mode 100644 src/AngleSharp.Io/Dom/ReportingObserver.cs create mode 100644 src/AngleSharp.Io/Dom/RequestBody.cs create mode 100644 src/AngleSharp.Io/Dom/ServiceWorker.cs create mode 100644 src/AngleSharp.Io/Dom/ServiceWorkerContainer.cs create mode 100644 src/AngleSharp.Io/Dom/ServiceWorkerRegistration.cs create mode 100644 src/AngleSharp.Io/Dom/ServiceWorkerState.cs create mode 100644 src/AngleSharp.Io/Dom/ServiceWorkerUpdateViaCache.cs create mode 100644 src/AngleSharp.Io/Dom/StorageManager.cs create mode 100644 src/AngleSharp.Io/Dom/WorkerType.cs create mode 100644 src/AngleSharp.Io/Interfaces/IPermissionPolicy.cs create mode 100644 src/AngleSharp.Io/Interfaces/IServiceWorkerPlatform.cs create mode 100644 src/AngleSharp.Io/Interfaces/IServiceWorkerProviderFactory.cs create mode 100644 src/AngleSharp.Io/Interfaces/IStorageManagerPlatform.cs create mode 100644 src/AngleSharp.Io/Interfaces/IStorageManagerProviderFactory.cs create mode 100644 src/AngleSharp.Io/Permissions/PermissionNames.cs create mode 100644 src/AngleSharp.Io/Permissions/PermissionPolicy.cs create mode 100644 src/AngleSharp.Io/Reporting/EndpointReportSink.cs create mode 100644 src/AngleSharp.Io/Reporting/IReportSink.cs create mode 100644 src/AngleSharp.Io/Reporting/MemoryReportSink.cs create mode 100644 src/AngleSharp.Io/Reporting/NetworkErrorReportBody.cs create mode 100644 src/AngleSharp.Io/Reporting/NetworkFailures.cs create mode 100644 src/AngleSharp.Io/Reporting/Report.cs create mode 100644 src/AngleSharp.Io/Reporting/ReportBody.cs create mode 100644 src/AngleSharp.Io/Reporting/ReportEventArgs.cs create mode 100644 src/AngleSharp.Io/Reporting/ReportFilter.cs create mode 100644 src/AngleSharp.Io/Reporting/ReportJson.cs create mode 100644 src/AngleSharp.Io/Reporting/ReportTypes.cs create mode 100644 src/AngleSharp.Io/Reporting/ReportingOptions.cs create mode 100644 src/AngleSharp.Io/Reporting/ReportingRequester.cs create mode 100644 src/AngleSharp.Io/Reporting/ReportingService.cs create mode 100644 src/AngleSharp.Io/ServiceWorker/ServiceWorkerPlatform.cs create mode 100644 src/AngleSharp.Io/ServiceWorker/ServiceWorkerProviderFactory.cs create mode 100644 src/AngleSharp.Io/ServiceWorker/ServiceWorkerRegistry.cs create mode 100644 src/AngleSharp.Io/ServiceWorker/ServiceWorkerRequester.cs create mode 100644 src/AngleSharp.Io/StorageManager/MemoryStorageManagerPlatform.cs create mode 100644 src/AngleSharp.Io/StorageManager/StorageManagerProviderFactory.cs create mode 100644 src/AngleSharp.Io/WebRtc/IRtcPlatform.cs create mode 100644 src/AngleSharp.Io/WebRtc/LoopbackRtcPlatform.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcBundlePolicy.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcConfiguration.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcDataChannel.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcDataChannelState.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcEvents.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcIceCandidate.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcIceCandidateType.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcIceComponent.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcIceConnectionState.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcIceGatheringState.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcIceProtocol.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcIceTcpCandidateType.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcIceTransportPolicy.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcPeerConnection.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcPeerConnectionState.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcRtcpMuxPolicy.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcSdpType.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcSessionDescription.cs create mode 100644 src/AngleSharp.Io/WebRtc/RtcSignalingState.cs diff --git a/AGENTS.md b/AGENTS.md index 7bbd2be..438ee44 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,6 +48,10 @@ The main integration surface is `src/AngleSharp.Io/IoConfigurationExtensions.cs` - `WithHttpCache(...)` wraps the registered HTTP(S) requesters in `CachingRequester` (`src/AngleSharp.Io/Http`), so it must be called after `WithRequesters(...)`. Not to be confused with `Cache/` (`window.caches`, the DOM Cache Storage API). +- `WithPermissions(...)` registers the `IPermissionPolicy` behind `navigator.permissions`. APIs + using a powerful feature take the navigator's `Permissions` (null = ungated) and call + `EnsureGrantedAsync` / `RequestAsync` instead of their own checks; add new names to + `PermissionNames` (`src/AngleSharp.Io/Permissions`). Cookie subsystem layout (`src/AngleSharp.Io/Cookie`): diff --git a/CHANGELOG.md b/CHANGELOG.md index bbe3bf1..6dfc93a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ Released on Wednesday, October 21 2026. - Added an HTTP cache for the requesters (`WithHttpCache`) - Added `IoOptions.CreateTemporary` and `IoOptions.CreatePersistent` +- Added `StorageManager` on `navigator.storage` (#38) +- Added minimal service workers on `navigator.serviceWorker` (#39) +- Added `navigator.sendBeacon` (#43) +- Added reporting with network error capture and `ReportingObserver` (#46) +- Added `RTCPeerConnection` with data channels and a loopback platform (#40) +- Added `navigator.permissions` with a policy gating clipboard, geolocation, and `persist` (#47) # 1.1.0 diff --git a/README.md b/README.md index 045ebe4..05ab150 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,12 @@ If you want to register all core AngleSharp.Io services in one step, use `WithIo var options = new IoOptions { // optional; defaults are already provided + Permissions = new PermissionPolicy(), ClipboardPlatform = myClipboardPlatform, GeolocationPlatform = myGeolocationPlatform, + StorageManagerPlatform = myStorageManagerPlatform, + ServiceWorkerPlatform = myServiceWorkerPlatform, + WebRtcPlatform = new LoopbackRtcPlatform(), }; var config = Configuration.Default @@ -30,7 +34,8 @@ var config = Configuration.Default ``` `WithIo` wires navigator, cookies, requesters, storage, IndexedDB, and cache in one call. -Clipboard and geolocation are only enabled when platforms are provided. +Clipboard, geolocation, the storage manager, service workers, and WebRTC are only enabled when platforms are provided. +The [permissions](#permissions) are only checked when a policy is provided. To keep all browsing data in one place, start from `IoOptions.CreateTemporary()` (memory only) or `IoOptions.CreatePersistent(rootDirectory)`, which stores the cookies (`cookies.txt`), the local @@ -299,6 +304,62 @@ await locks.Request("app", async _ => }); ``` +### Permissions + +AngleSharp.Io provides `navigator.permissions` backed by a single permission policy. The same +policy decides whether the powerful features may be used, so there is one place to configure them: + +- `permissions` exposing `query`, which returns a `PermissionStatus` with `name`, `state`, and `onchange` +- `clipboard.readText` / `clipboard.writeText` check `clipboard-read` / `clipboard-write` +- `geolocation.getCurrentPosition` checks `geolocation` +- `storage.persist` checks `persistent-storage` (and resolves with `false` instead of rejecting) + +The names in `PermissionNames` are known by default (`notifications` can be queried, but nothing +uses it yet). A denied permission rejects with a `DomException` named `NotAllowedError`. + +`PermissionPolicy` keeps the states in memory. Every permission starts as `prompt`, and as no user +can answer the prompt in a headless browser, the feature is denied until you grant the permission: + +```cs +var policy = new PermissionPolicy(); + +// for all origins; entries for a single origin take precedence +policy.Set(PermissionNames.ClipboardWrite, PermissionState.Granted); +policy.Set("https://example.com", PermissionNames.Geolocation, PermissionState.Granted); + +// optional: answer prompts in place of the user, the answer is remembered for the origin +policy.PromptHandler = (origin, name) => Task.FromResult(PermissionState.Denied); + +// optional: make a permission of another library known +policy.Register("local-fonts"); + +var config = Configuration.Default + .WithNavigator() + .WithPermissions(policy) + .WithClipboard(new MyClipboardPlatform()); +``` + +Use `new PermissionPolicy(PermissionState.Granted)` to allow everything, or implement +`IPermissionPolicy` to take the decisions from somewhere else. Without `WithPermissions` there is +no `navigator.permissions` and the features are not gated, as in previous versions. + +Statuses follow later changes of the policy and dispatch `change` when their state changes: + +```cs +var permissions = document.DefaultView.Navigator.Permissions(); +var status = await permissions.Query(new PermissionDescriptor { Name = "geolocation" }); +status.Changed += (sender, ev) => Console.WriteLine(status.State); + +policy.Set("https://example.com", PermissionNames.Geolocation, PermissionState.Denied); +``` + +APIs from other libraries can use the same check before touching a feature: + +```cs +await permissions.EnsureGrantedAsync("local-fonts"); // throws NotAllowedError if not granted +var state = await permissions.RequestAsync("local-fonts"); // granted or denied, never prompt +``` + ### Clipboard AngleSharp.Io provides a navigator clipboard surface backed by a user-supplied platform implementation: @@ -377,6 +438,268 @@ var geolocation = navigator.Geolocation(); var position = await geolocation.GetCurrentPosition(); ``` +### Storage Manager + +AngleSharp.Io provides `navigator.storage` backed by a user-supplied platform implementation: + +- `storage` exposing `estimate`, `persist`, and `persisted` + +The platform receives the origin of the active document. `MemoryStorageManagerPlatform` reports +fixed values and keeps the persistence state in memory, which makes it a good fit for tests: + +```cs +var platform = new MemoryStorageManagerPlatform +{ + Quota = 1024 * 1024 * 1024, + Usage = 4096, + AllowPersistence = true, // the default +}; + +var config = Configuration.Default + .WithNavigator() + .WithStorageManager(platform); +``` + +For host-specific behavior implement `IStorageManagerPlatform` instead. Resolve it from `INavigator`: + +```cs +var storage = document.DefaultView.Navigator.Storage(); +var estimate = await storage.Estimate(); // estimate.Usage, estimate.Quota +var granted = await storage.Persist(); +var persisted = await storage.Persisted(); +``` + +### Service Workers + +AngleSharp.Io provides `navigator.serviceWorker` with a minimal registration surface: + +- `serviceWorker` exposing `register` (with `scope`, `type`, and `updateViaCache`), `getRegistration`, + `getRegistrations`, `ready`, `controller`, and the `controllerchange` and `message` events +- registrations exposing `scope`, `updateViaCache`, `installing`, `waiting`, `active`, `update`, + `unregister`, and the `updatefound` event +- workers exposing `scriptURL`, `state`, `postMessage`, and the `statechange` event + +As in browsers, `navigator.serviceWorker` is only available in secure contexts, i.e., for `https` +documents and for `http` documents served from `localhost` or a loopback address. + +Running the worker script is up to an `IServiceWorkerPlatform`. The included `ServiceWorkerPlatform` +delegates to optional callbacks; without callbacks every worker installs and activates and all +requests go to the network: + +```cs +var platform = new ServiceWorkerPlatform +{ + Install = worker => Task.CompletedTask, // a faulted task makes the worker redundant + CheckForUpdate = worker => Task.FromResult(false), // true installs a new worker + Message = (worker, message) => { }, // receives worker.postMessage(...) + Fetch = (worker, request, cancel) => + { + // return null to continue with the network + var response = VirtualResponse.Create(res => res.Address(request.Address).Content("offline")); + return Task.FromResult(response); + }, +}; + +var config = Configuration.Default + .WithNavigator() + .WithRequesters() + .WithServiceWorkers(platform) // after WithRequesters + .WithDefaultLoader(); +``` + +`WithServiceWorkers` wraps the registered HTTP(S) requesters, so it must be called after +`WithRequesters`. Requests (including navigations and `fetch`) whose URL lies within the scope of +an active worker are passed to the platform first; the HTTP cache, if any, stays behind the worker. + +```cs +var container = document.DefaultView.Navigator.ServiceWorker(); +var registration = await container.Register("/app/sw.js"); // scope defaults to /app/ +var state = registration.Active.State; // ServiceWorkerState.Activated +var ready = await container.Ready; // registration controlling the document +await registration.Update(); // asks the platform's CheckForUpdate +``` + +Workers reply to documents via the registry, which dispatches a `message` event on the containers +of the documents they control: + +```cs +var registry = context.GetService(); +registry.PostMessageToClients(registration.Active, "hello"); +``` + +Compared to browsers the lifecycle is simplified: `register` and `update` complete once the worker +is activated (or redundant if its installation failed), new workers are activated right away as if +`skipWaiting()` was called, and `unregister` drops the workers immediately. Since scripts are not +fetched, the platform decides whether a script changed (`CheckForUpdate`) and the scope cannot be +wider than the script's directory. Requests are matched by their own URL and the controller is +determined on access, as clients are not tracked. + +### Beacon + +AngleSharp.Io provides `navigator.sendBeacon` for fire-and-forget requests, e.g., for telemetry: + +- `sendBeacon(url, data)` queues a `POST` request and returns right away + +The data can be a string (sent as `text/plain;charset=UTF-8`), a blob (sent with its type), a byte +array or `ArraySegment`, `FormDataSet` (sent as `multipart/form-data`), or `UrlSearchParams` +(sent as `application/x-www-form-urlencoded`). The request goes through the configured +`IDocumentLoader`, i.e., the requesters and cookies apply as for `fetch`. + +```cs +var config = Configuration.Default + .WithNavigator() + .WithRequesters() + .WithDefaultLoader(); + +var navigator = document.DefaultView.Navigator; +var queued = navigator.SendBeacon("/collect", "session ended"); +``` + +`SendBeacon` returns `false` if the beacon was not queued: when no loader is registered, when the +data is not supported (e.g., a `Stream`, whose length is unknown upfront) or cannot be serialized, +or when the body would exceed the quota. As in browsers, the bodies of the pending beacons may not +exceed 64 KiB per browsing context. Once queued, failures (e.g., network errors) are not reported, +but release the quota. Beacons keep going when their document navigates away, closes, or its +browsing context is disposed. + +`BeaconOptions` changes the quota and can limit the number of beacons sent at the same time; further +beacons are queued, count towards the quota, and are sent in order: + +```cs +var config = Configuration.Default + .WithNavigator() + .WithRequesters() + .WithDefaultLoader() + .WithBeacon(new BeaconOptions + { + Quota = 256 * 1024, + MaxConcurrentRequests = 4, // 0 (default) sends all right away + }); +``` + +Beacons carry an `Origin` header. Like in browsers, a beacon to another origin whose content type is +not `text/plain`, `application/x-www-form-urlencoded`, or `multipart/form-data` (e.g., a JSON blob) +is preceded by a CORS preflight (`OPTIONS`, without cookies). The beacon is only sent if the response +allows the origin (`Access-Control-Allow-Origin` with the exact origin), credentials +(`Access-Control-Allow-Credentials: true`), and the `content-type` header +(`Access-Control-Allow-Headers`). Preflight results are not cached. + +An invalid or non-HTTP(S) URL throws an `InvalidOperationException`, standing in for the +`TypeError` of the spec. + +### Reporting + +AngleSharp.Io collects browser-like diagnostics as reports, following the shape of the Reporting +API and Network Error Logging (NEL): + +- `WithReporting` registers a `ReportingService` that queues reports until they are flushed +- failed requests and responses with a status of 400 or above are captured as `network-error` + reports from the requester pipeline +- `window.QueueReport(...)` emits reports about a document, e.g., deprecations +- `ReportingObserver` exposes the reports of a window (`observe`, `disconnect`, `takeRecords`) + +Flushed reports go to the configured sinks, each with an optional filter by type and endpoint: + +```cs +var memory = new MemoryReportSink(); +var options = new ReportingOptions() + .AddSink(memory) + .AddSink(new EndpointReportSink(new HttpClientRequester()), ReportFilter.ForTypes(ReportTypes.NetworkError)) + .AddSink((reports, cancel) => LogAsync(reports), ReportFilter.ForDestinations("default")); + +options.Endpoints["default"] = "https://collector.example/reports"; + +var config = Configuration.Default + .WithRequesters() + .WithReporting(options) // after WithRequesters + .WithDefaultLoader(); +``` + +The service can be resolved from the configuration or a browsing context to enumerate, emit, and +flush the reports: + +```cs +var reporting = context.GetService(); +var errors = reporting.GetReports(ReportFilter.ForTypes(ReportTypes.NetworkError)); +reporting.QueueReport(new Report("custom", "https://example.org/", new GenericReportBody(members))); +await reporting.FlushAsync(); // removes the queued reports and hands them to the sinks +``` + +Reports serialize to JSON via `ToJson()` (`type`, `url`, `body`) and `ToDeliveryJson(now)`, the +format posted by `EndpointReportSink` as `application/reports+json` (adding `age` and `user_agent`). +A `network-error` report carries a `NetworkErrorReportBody` with the NEL members, e.g., `phase` +(`dns`, `connection`, `application`), `type` (e.g., `dns.name_not_resolved`, `tcp.refused`, +`tcp.timed_out`, `tls.failed`, `abandoned`, `http.error`), `elapsed_time`, `method`, `referrer`, +and `status_code`. Report URLs exclude credentials and fragments. + +Further options: + +- `Filter` decides which reports are queued at all +- `CaptureNetworkErrors` / `CaptureHttpErrors` turn the captured network errors on or off +- `NetworkErrorDestination` names the endpoint of network errors (`default` by default) +- `MaxQueueSize` limits the queue, dropping the oldest reports (1000 by default) +- `FlushInterval` flushes automatically, otherwise reports are only flushed by `FlushAsync` + +Endpoint names are resolved to URLs via the `Reporting-Endpoints` header of responses from the +report's origin, falling back to `Endpoints`. The reporting sits closest to the network, i.e., +behind the HTTP cache and service workers, so only requests reaching the network are reported. + +As in browsers, a `ReportingObserver` callback runs as a task of the browsing context's event loop +(`IEventLoop`, e.g., provided by a scripting engine), receiving all reports added until the task +runs; without an event loop the callback is invoked right away. + +Compared to browsers this is a lightweight subset: a flush removes the reports even if a sink fails +(the failure is rethrown), and `NEL` policy headers are not interpreted, i.e., network errors of +every origin are captured. + +### WebRTC + +AngleSharp.Io provides the signaling part of WebRTC via `RTCPeerConnection`, with the transport +provided by a pluggable `IRtcPlatform`: + +- `createOffer`, `createAnswer`, `setLocalDescription` (also without arguments), and + `setRemoteDescription`, including provisional answers and (implicit) rollbacks +- `addIceCandidate`, `restartIce`, `getConfiguration`, `setConfiguration`, and `close` +- the `signalingState`, `iceGatheringState`, `iceConnectionState`, and `connectionState` + attributes with their change events, plus `negotiationneeded` and `icecandidate` +- `createDataChannel` with `RTCDataChannel` (`send`, `close`, and the `open`, `message`, + `closing`, `close` events) and the `datachannel` event + +`WithWebRtc()` registers the `LoopbackRtcPlatform`, which connects the peer connections of the +configuration with each other in-process: its SDP is deterministic, each peer gathers a single host +candidate on 127.0.0.1, and a peer connects once both descriptions are applied and it received a +candidate of the other peer. Data channel messages are delivered between the connected peers. + +```cs +var config = Configuration.Default.WithWebRtc(); +var context = BrowsingContext.New(config); +var document = await context.OpenAsync("https://example.org"); + +var caller = new RtcPeerConnection(document.DefaultView); +var callee = new RtcPeerConnection(document.DefaultView); +var channel = caller.CreateDataChannel("chat"); + +var offer = await caller.CreateOffer(); +await caller.SetLocalDescription(offer); +await callee.SetRemoteDescription(offer); +var answer = await callee.CreateAnswer(); +await callee.SetLocalDescription(answer); +await caller.SetRemoteDescription(answer); +// then exchange the candidates of the icecandidate events via addIceCandidate +``` + +A custom `IRtcPlatform` (e.g., backed by a WebRTC stack or a test double) creates an `IRtcPeer` per +connection, which produces the SDP, applies the descriptions, and reports candidates, state changes, +and data channel events through the given `IRtcPeerHost`. Without a registered platform, +constructing an `RTCPeerConnection` from a window fails with `NotSupportedError`; the constructor +taking an `IRtcPlatform` works without any configuration. + +As in browsers, the events of the transport (candidates, ICE states, data channels) are dispatched as +tasks of the browsing context's event loop. Without an event loop they are dispatched once the +running operation completed, i.e., always after the state change they belong to. Media (tracks, +transceivers, codecs) and statistics are not supported; data channels deliver messages in order +without modeling the reliability parameters, and binary messages are always byte arrays. + ### Fetch AngleSharp.Io also exposes a `fetch` method on `window`: @@ -461,8 +784,14 @@ The `SaveToAsync` (as well as the `CopyToAsync`) are extension methods for the ` - Private HTTP cache for the requesters (memory or persisted) - Enhanced download capabilities for resources / links - Web Locks support for origin-scoped request serialization +- Permissions (`navigator.permissions`) with one injectable policy gating the powerful features - Clipboard support with injectable platform implementation - Geolocation support with injectable platform implementation +- Storage manager (`navigator.storage`) support with injectable platform implementation +- Service worker registrations (`navigator.serviceWorker`) with request interception +- Beacon support (`navigator.sendBeacon`) for fire-and-forget requests +- Reporting of network errors and other diagnostics with injectable sinks +- WebRTC signaling (`RTCPeerConnection`) with data channels and a pluggable transport ## Participating diff --git a/docs/general/01-Basics.md b/docs/general/01-Basics.md index 4700a73..be97301 100644 --- a/docs/general/01-Basics.md +++ b/docs/general/01-Basics.md @@ -38,8 +38,12 @@ If you want to register all core AngleSharp.Io services in one step, use `WithIo var options = new IoOptions { // optional; defaults are already provided + Permissions = new PermissionPolicy(), ClipboardPlatform = myClipboardPlatform, GeolocationPlatform = myGeolocationPlatform, + StorageManagerPlatform = myStorageManagerPlatform, + ServiceWorkerPlatform = myServiceWorkerPlatform, + WebRtcPlatform = new LoopbackRtcPlatform(), }; var config = Configuration.Default @@ -48,7 +52,8 @@ var config = Configuration.Default ``` `WithIo` wires navigator, cookies, requesters, storage, IndexedDB, and cache in one call. -Clipboard and geolocation are only enabled when platforms are provided. +Clipboard, geolocation, the storage manager, service workers, and WebRTC are only enabled when platforms are provided. +The [permissions](#permissions) are only checked when a policy is provided. Instead of `new IoOptions()` you can start from one of the factory methods, which also turn on the [HTTP cache](#http-cache): @@ -400,6 +405,62 @@ await locks.Request("app", async _ => }); ``` +### Permissions + +AngleSharp.Io provides `navigator.permissions` backed by a single permission policy. The same +policy decides whether the powerful features may be used, so there is one place to configure them: + +- `permissions` exposing `query`, which returns a `PermissionStatus` with `name`, `state`, and `onchange` +- `clipboard.readText` / `clipboard.writeText` check `clipboard-read` / `clipboard-write` +- `geolocation.getCurrentPosition` checks `geolocation` +- `storage.persist` checks `persistent-storage` (and resolves with `false` instead of rejecting) + +The names in `PermissionNames` are known by default (`notifications` can be queried, but nothing +uses it yet). A denied permission rejects with a `DomException` named `NotAllowedError`. + +`PermissionPolicy` keeps the states in memory. Every permission starts as `prompt`, and as no user +can answer the prompt in a headless browser, the feature is denied until you grant the permission: + +```cs +var policy = new PermissionPolicy(); + +// for all origins; entries for a single origin take precedence +policy.Set(PermissionNames.ClipboardWrite, PermissionState.Granted); +policy.Set("https://example.com", PermissionNames.Geolocation, PermissionState.Granted); + +// optional: answer prompts in place of the user, the answer is remembered for the origin +policy.PromptHandler = (origin, name) => Task.FromResult(PermissionState.Denied); + +// optional: make a permission of another library known +policy.Register("local-fonts"); + +var config = Configuration.Default + .WithNavigator() + .WithPermissions(policy) + .WithClipboard(new MyClipboardPlatform()); +``` + +Use `new PermissionPolicy(PermissionState.Granted)` to allow everything, or implement +`IPermissionPolicy` to take the decisions from somewhere else. Without `WithPermissions` there is +no `navigator.permissions` and the features are not gated, as in previous versions. + +Statuses follow later changes of the policy and dispatch `change` when their state changes: + +```cs +var permissions = document.DefaultView.Navigator.Permissions(); +var status = await permissions.Query(new PermissionDescriptor { Name = "geolocation" }); +status.Changed += (sender, ev) => Console.WriteLine(status.State); + +policy.Set("https://example.com", PermissionNames.Geolocation, PermissionState.Denied); +``` + +APIs from other libraries can use the same check before touching a feature: + +```cs +await permissions.EnsureGrantedAsync("local-fonts"); // throws NotAllowedError if not granted +var state = await permissions.RequestAsync("local-fonts"); // granted or denied, never prompt +``` + ### Clipboard AngleSharp.Io provides a navigator clipboard surface backed by a user-supplied platform implementation: @@ -478,6 +539,268 @@ var geolocation = navigator.Geolocation(); var position = await geolocation.GetCurrentPosition(); ``` +### Storage Manager + +AngleSharp.Io provides `navigator.storage` backed by a user-supplied platform implementation: + +- `storage` exposing `estimate`, `persist`, and `persisted` + +The platform receives the origin of the active document. `MemoryStorageManagerPlatform` reports +fixed values and keeps the persistence state in memory, which makes it a good fit for tests: + +```cs +var platform = new MemoryStorageManagerPlatform +{ + Quota = 1024 * 1024 * 1024, + Usage = 4096, + AllowPersistence = true, // the default +}; + +var config = Configuration.Default + .WithNavigator() + .WithStorageManager(platform); +``` + +For host-specific behavior implement `IStorageManagerPlatform` instead. Resolve it from `INavigator`: + +```cs +var storage = document.DefaultView.Navigator.Storage(); +var estimate = await storage.Estimate(); // estimate.Usage, estimate.Quota +var granted = await storage.Persist(); +var persisted = await storage.Persisted(); +``` + +### Service Workers + +AngleSharp.Io provides `navigator.serviceWorker` with a minimal registration surface: + +- `serviceWorker` exposing `register` (with `scope`, `type`, and `updateViaCache`), `getRegistration`, + `getRegistrations`, `ready`, `controller`, and the `controllerchange` and `message` events +- registrations exposing `scope`, `updateViaCache`, `installing`, `waiting`, `active`, `update`, + `unregister`, and the `updatefound` event +- workers exposing `scriptURL`, `state`, `postMessage`, and the `statechange` event + +As in browsers, `navigator.serviceWorker` is only available in secure contexts, i.e., for `https` +documents and for `http` documents served from `localhost` or a loopback address. + +Running the worker script is up to an `IServiceWorkerPlatform`. The included `ServiceWorkerPlatform` +delegates to optional callbacks; without callbacks every worker installs and activates and all +requests go to the network: + +```cs +var platform = new ServiceWorkerPlatform +{ + Install = worker => Task.CompletedTask, // a faulted task makes the worker redundant + CheckForUpdate = worker => Task.FromResult(false), // true installs a new worker + Message = (worker, message) => { }, // receives worker.postMessage(...) + Fetch = (worker, request, cancel) => + { + // return null to continue with the network + var response = VirtualResponse.Create(res => res.Address(request.Address).Content("offline")); + return Task.FromResult(response); + }, +}; + +var config = Configuration.Default + .WithNavigator() + .WithRequesters() + .WithServiceWorkers(platform) // after WithRequesters + .WithDefaultLoader(); +``` + +`WithServiceWorkers` wraps the registered HTTP(S) requesters, so it must be called after +`WithRequesters`. Requests (including navigations and `fetch`) whose URL lies within the scope of +an active worker are passed to the platform first; the HTTP cache, if any, stays behind the worker. + +```cs +var container = document.DefaultView.Navigator.ServiceWorker(); +var registration = await container.Register("/app/sw.js"); // scope defaults to /app/ +var state = registration.Active.State; // ServiceWorkerState.Activated +var ready = await container.Ready; // registration controlling the document +await registration.Update(); // asks the platform's CheckForUpdate +``` + +Workers reply to documents via the registry, which dispatches a `message` event on the containers +of the documents they control: + +```cs +var registry = context.GetService(); +registry.PostMessageToClients(registration.Active, "hello"); +``` + +Compared to browsers the lifecycle is simplified: `register` and `update` complete once the worker +is activated (or redundant if its installation failed), new workers are activated right away as if +`skipWaiting()` was called, and `unregister` drops the workers immediately. Since scripts are not +fetched, the platform decides whether a script changed (`CheckForUpdate`) and the scope cannot be +wider than the script's directory. Requests are matched by their own URL and the controller is +determined on access, as clients are not tracked. + +### Beacon + +AngleSharp.Io provides `navigator.sendBeacon` for fire-and-forget requests, e.g., for telemetry: + +- `sendBeacon(url, data)` queues a `POST` request and returns right away + +The data can be a string (sent as `text/plain;charset=UTF-8`), a blob (sent with its type), a byte +array or `ArraySegment`, `FormDataSet` (sent as `multipart/form-data`), or `UrlSearchParams` +(sent as `application/x-www-form-urlencoded`). The request goes through the configured +`IDocumentLoader`, i.e., the requesters and cookies apply as for `fetch`. + +```cs +var config = Configuration.Default + .WithNavigator() + .WithRequesters() + .WithDefaultLoader(); + +var navigator = document.DefaultView.Navigator; +var queued = navigator.SendBeacon("/collect", "session ended"); +``` + +`SendBeacon` returns `false` if the beacon was not queued: when no loader is registered, when the +data is not supported (e.g., a `Stream`, whose length is unknown upfront) or cannot be serialized, +or when the body would exceed the quota. As in browsers, the bodies of the pending beacons may not +exceed 64 KiB per browsing context. Once queued, failures (e.g., network errors) are not reported, +but release the quota. Beacons keep going when their document navigates away, closes, or its +browsing context is disposed. + +`BeaconOptions` changes the quota and can limit the number of beacons sent at the same time; further +beacons are queued, count towards the quota, and are sent in order: + +```cs +var config = Configuration.Default + .WithNavigator() + .WithRequesters() + .WithDefaultLoader() + .WithBeacon(new BeaconOptions + { + Quota = 256 * 1024, + MaxConcurrentRequests = 4, // 0 (default) sends all right away + }); +``` + +Beacons carry an `Origin` header. Like in browsers, a beacon to another origin whose content type is +not `text/plain`, `application/x-www-form-urlencoded`, or `multipart/form-data` (e.g., a JSON blob) +is preceded by a CORS preflight (`OPTIONS`, without cookies). The beacon is only sent if the response +allows the origin (`Access-Control-Allow-Origin` with the exact origin), credentials +(`Access-Control-Allow-Credentials: true`), and the `content-type` header +(`Access-Control-Allow-Headers`). Preflight results are not cached. + +An invalid or non-HTTP(S) URL throws an `InvalidOperationException`, standing in for the +`TypeError` of the spec. + +### Reporting + +AngleSharp.Io collects browser-like diagnostics as reports, following the shape of the Reporting +API and Network Error Logging (NEL): + +- `WithReporting` registers a `ReportingService` that queues reports until they are flushed +- failed requests and responses with a status of 400 or above are captured as `network-error` + reports from the requester pipeline +- `window.QueueReport(...)` emits reports about a document, e.g., deprecations +- `ReportingObserver` exposes the reports of a window (`observe`, `disconnect`, `takeRecords`) + +Flushed reports go to the configured sinks, each with an optional filter by type and endpoint: + +```cs +var memory = new MemoryReportSink(); +var options = new ReportingOptions() + .AddSink(memory) + .AddSink(new EndpointReportSink(new HttpClientRequester()), ReportFilter.ForTypes(ReportTypes.NetworkError)) + .AddSink((reports, cancel) => LogAsync(reports), ReportFilter.ForDestinations("default")); + +options.Endpoints["default"] = "https://collector.example/reports"; + +var config = Configuration.Default + .WithRequesters() + .WithReporting(options) // after WithRequesters + .WithDefaultLoader(); +``` + +The service can be resolved from the configuration or a browsing context to enumerate, emit, and +flush the reports: + +```cs +var reporting = context.GetService(); +var errors = reporting.GetReports(ReportFilter.ForTypes(ReportTypes.NetworkError)); +reporting.QueueReport(new Report("custom", "https://example.org/", new GenericReportBody(members))); +await reporting.FlushAsync(); // removes the queued reports and hands them to the sinks +``` + +Reports serialize to JSON via `ToJson()` (`type`, `url`, `body`) and `ToDeliveryJson(now)`, the +format posted by `EndpointReportSink` as `application/reports+json` (adding `age` and `user_agent`). +A `network-error` report carries a `NetworkErrorReportBody` with the NEL members, e.g., `phase` +(`dns`, `connection`, `application`), `type` (e.g., `dns.name_not_resolved`, `tcp.refused`, +`tcp.timed_out`, `tls.failed`, `abandoned`, `http.error`), `elapsed_time`, `method`, `referrer`, +and `status_code`. Report URLs exclude credentials and fragments. + +Further options: + +- `Filter` decides which reports are queued at all +- `CaptureNetworkErrors` / `CaptureHttpErrors` turn the captured network errors on or off +- `NetworkErrorDestination` names the endpoint of network errors (`default` by default) +- `MaxQueueSize` limits the queue, dropping the oldest reports (1000 by default) +- `FlushInterval` flushes automatically, otherwise reports are only flushed by `FlushAsync` + +Endpoint names are resolved to URLs via the `Reporting-Endpoints` header of responses from the +report's origin, falling back to `Endpoints`. The reporting sits closest to the network, i.e., +behind the HTTP cache and service workers, so only requests reaching the network are reported. + +As in browsers, a `ReportingObserver` callback runs as a task of the browsing context's event loop +(`IEventLoop`, e.g., provided by a scripting engine), receiving all reports added until the task +runs; without an event loop the callback is invoked right away. + +Compared to browsers this is a lightweight subset: a flush removes the reports even if a sink fails +(the failure is rethrown), and `NEL` policy headers are not interpreted, i.e., network errors of +every origin are captured. + +### WebRTC + +AngleSharp.Io provides the signaling part of WebRTC via `RTCPeerConnection`, with the transport +provided by a pluggable `IRtcPlatform`: + +- `createOffer`, `createAnswer`, `setLocalDescription` (also without arguments), and + `setRemoteDescription`, including provisional answers and (implicit) rollbacks +- `addIceCandidate`, `restartIce`, `getConfiguration`, `setConfiguration`, and `close` +- the `signalingState`, `iceGatheringState`, `iceConnectionState`, and `connectionState` + attributes with their change events, plus `negotiationneeded` and `icecandidate` +- `createDataChannel` with `RTCDataChannel` (`send`, `close`, and the `open`, `message`, + `closing`, `close` events) and the `datachannel` event + +`WithWebRtc()` registers the `LoopbackRtcPlatform`, which connects the peer connections of the +configuration with each other in-process: its SDP is deterministic, each peer gathers a single host +candidate on 127.0.0.1, and a peer connects once both descriptions are applied and it received a +candidate of the other peer. Data channel messages are delivered between the connected peers. + +```cs +var config = Configuration.Default.WithWebRtc(); +var context = BrowsingContext.New(config); +var document = await context.OpenAsync("https://example.org"); + +var caller = new RtcPeerConnection(document.DefaultView); +var callee = new RtcPeerConnection(document.DefaultView); +var channel = caller.CreateDataChannel("chat"); + +var offer = await caller.CreateOffer(); +await caller.SetLocalDescription(offer); +await callee.SetRemoteDescription(offer); +var answer = await callee.CreateAnswer(); +await callee.SetLocalDescription(answer); +await caller.SetRemoteDescription(answer); +// then exchange the candidates of the icecandidate events via addIceCandidate +``` + +A custom `IRtcPlatform` (e.g., backed by a WebRTC stack or a test double) creates an `IRtcPeer` per +connection, which produces the SDP, applies the descriptions, and reports candidates, state changes, +and data channel events through the given `IRtcPeerHost`. Without a registered platform, +constructing an `RTCPeerConnection` from a window fails with `NotSupportedError`; the constructor +taking an `IRtcPlatform` works without any configuration. + +As in browsers, the events of the transport (candidates, ICE states, data channels) are dispatched as +tasks of the browsing context's event loop. Without an event loop they are dispatched once the +running operation completed, i.e., always after the state change they belong to. Media (tracks, +transceivers, codecs) and statistics are not supported; data channels deliver messages in order +without modeling the reliability parameters, and binary messages are always byte arrays. + ### Fetch AngleSharp.Io also exposes a `fetch` method on `window`: diff --git a/docs/tutorials/03-Common-Workflows.md b/docs/tutorials/03-Common-Workflows.md index e7fdb97..362f2bb 100644 --- a/docs/tutorials/03-Common-Workflows.md +++ b/docs/tutorials/03-Common-Workflows.md @@ -15,6 +15,9 @@ var options = new IoOptions { ClipboardPlatform = myClipboardPlatform, // optional GeolocationPlatform = myGeolocationPlatform, // optional + StorageManagerPlatform = myStorageManagerPlatform, // optional + ServiceWorkerPlatform = myServiceWorkerPlatform, // optional + WebRtcPlatform = new LoopbackRtcPlatform(), // optional }; var config = Configuration.Default diff --git a/src/AngleSharp.Io.Tests/Beacon/BeaconTests.cs b/src/AngleSharp.Io.Tests/Beacon/BeaconTests.cs new file mode 100644 index 0000000..655dec7 --- /dev/null +++ b/src/AngleSharp.Io.Tests/Beacon/BeaconTests.cs @@ -0,0 +1,655 @@ +namespace AngleSharp.Io.Tests.Beacon +{ + using AngleSharp; + using AngleSharp.Browser.Dom; + using AngleSharp.Dom; + using AngleSharp.Html; + using AngleSharp.Io; + using AngleSharp.Io.Dom; + using NUnit.Framework; + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + + [TestFixture] + public class BeaconTests + { + [Test] + public async Task SendBeaconPostsTextWithPlainTextContentType() + { + var loader = new TestDocumentLoader(); + var navigator = await OpenNavigatorAsync(loader, "https://beacon.example/app/page").ConfigureAwait(false); + + var result = navigator.SendBeacon("/collect?x=1", "hello"); + + Assert.IsTrue(result); + var request = loader.Requests.Single(); + Assert.AreEqual(HttpMethod.Post, request.Method); + Assert.AreEqual("https://beacon.example/collect?x=1", request.Target.Href); + Assert.AreEqual("https://beacon.example/app/page", request.Referer); + Assert.AreEqual("text/plain;charset=UTF-8", request.ContentType); + Assert.AreEqual("https://beacon.example", request.Origin); + Assert.AreEqual("hello", Encoding.UTF8.GetString(request.Body)); + } + + [Test] + public async Task SendBeaconWithoutDataPostsAnEmptyBody() + { + var loader = new TestDocumentLoader(); + var navigator = await OpenNavigatorAsync(loader, "https://beacon.example/").ConfigureAwait(false); + + Assert.IsTrue(navigator.SendBeacon("collect")); + Assert.IsTrue(navigator.SendBeacon("collect", null)); + + Assert.AreEqual(2, loader.Requests.Count); + Assert.IsTrue(loader.Requests.All(request => request.Method == HttpMethod.Post && request.Body.Length == 0 && request.ContentType is null)); + } + + [Test] + public async Task SendBeaconPostsBytesWithoutContentType() + { + var loader = new TestDocumentLoader(); + var navigator = await OpenNavigatorAsync(loader, "https://beacon.example/").ConfigureAwait(false); + + Assert.IsTrue(navigator.SendBeacon("/collect", new Byte[] { 1, 2, 3 })); + Assert.IsTrue(navigator.SendBeacon("/collect", new ArraySegment(new Byte[] { 1, 2, 3, 4, 5 }, 1, 3))); + + CollectionAssert.AreEqual(new Byte[] { 1, 2, 3 }, loader.Requests[0].Body); + CollectionAssert.AreEqual(new Byte[] { 2, 3, 4 }, loader.Requests[1].Body); + Assert.IsNull(loader.Requests[0].ContentType); + Assert.IsNull(loader.Requests[1].ContentType); + } + + [Test] + public async Task SendBeaconPostsBlobsWithTheirType() + { + var loader = new TestDocumentLoader(); + var navigator = await OpenNavigatorAsync(loader, "https://beacon.example/").ConfigureAwait(false); + var json = new InputFile("data.json", "application/json", Encoding.UTF8.GetBytes("{\"a\":1}")); + var untyped = new InputFile("data.bin", String.Empty, new Byte[] { 7 }); + + Assert.IsTrue(navigator.SendBeacon("/collect", json)); + Assert.IsTrue(navigator.SendBeacon("/collect", untyped)); + + Assert.AreEqual("application/json", loader.Requests[0].ContentType); + Assert.AreEqual("{\"a\":1}", Encoding.UTF8.GetString(loader.Requests[0].Body)); + Assert.IsNull(loader.Requests[1].ContentType); + CollectionAssert.AreEqual(new Byte[] { 7 }, loader.Requests[1].Body); + } + + [Test] + public async Task SendBeaconPostsFormDataAsMultipart() + { + var loader = new TestDocumentLoader(); + var navigator = await OpenNavigatorAsync(loader, "https://beacon.example/").ConfigureAwait(false); + var formData = new FormDataSet(); + formData.Append("name", "Ada", "text/plain"); + + Assert.IsTrue(navigator.SendBeacon("/collect", formData)); + + var request = loader.Requests.Single(); + Assert.AreEqual("multipart/form-data; boundary=" + formData.Boundary, request.ContentType); + StringAssert.Contains("Ada", Encoding.UTF8.GetString(request.Body)); + } + + [Test] + public async Task SendBeaconPostsSearchParamsAsUrlEncoded() + { + var loader = new TestDocumentLoader(); + var navigator = await OpenNavigatorAsync(loader, "https://beacon.example/").ConfigureAwait(false); + var searchParams = new UrlSearchParams("a=1&b=two"); + + Assert.IsTrue(navigator.SendBeacon("/collect", searchParams)); + + var request = loader.Requests.Single(); + StringAssert.StartsWith("application/x-www-form-urlencoded", request.ContentType); + Assert.AreEqual("a=1&b=two", Encoding.UTF8.GetString(request.Body)); + } + + [Test] + public async Task SendBeaconRejectsStreams() + { + var loader = new TestDocumentLoader(); + var navigator = await OpenNavigatorAsync(loader, "https://beacon.example/").ConfigureAwait(false); + + Assert.IsFalse(navigator.SendBeacon("/collect", new MemoryStream(new Byte[] { 1 }))); + Assert.IsEmpty(loader.Requests); + } + + [Test] + public async Task SendBeaconRejectsDataThatCannotBeSerialized() + { + var loader = new TestDocumentLoader(); + var navigator = await OpenNavigatorAsync(loader, "https://beacon.example/").ConfigureAwait(false); + + Assert.IsFalse(navigator.SendBeacon("/collect", new UnserializableData())); + Assert.IsEmpty(loader.Requests); + } + + [Test] + public async Task SendBeaconRejectsInvalidUrls() + { + var loader = new TestDocumentLoader(); + var navigator = await OpenNavigatorAsync(loader, "https://beacon.example/").ConfigureAwait(false); + + Assert.Throws(() => navigator.SendBeacon("ftp://beacon.example/collect", "x")); + Assert.Throws(() => navigator.SendBeacon("data:text/plain,x", "x")); + Assert.Throws(() => navigator.SendBeacon("http://[invalid/", "x")); + Assert.IsEmpty(loader.Requests); + } + + [Test] + public async Task SendBeaconRejectsPayloadsExceedingTheQuotaInFlight() + { + var loader = new TestDocumentLoader { Complete = false }; + var navigator = await OpenNavigatorAsync(loader, "https://beacon.example/").ConfigureAwait(false); + + Assert.IsTrue(navigator.SendBeacon("/collect", new Byte[40 * 1024])); + Assert.IsFalse(navigator.SendBeacon("/collect", new Byte[30 * 1024])); + Assert.IsTrue(navigator.SendBeacon("/collect", new Byte[24 * 1024])); + Assert.IsTrue(navigator.SendBeacon("/collect")); + Assert.IsFalse(navigator.SendBeacon("/collect", "x")); + + loader.Requests[0].Finish(); + await WaitForAsync(() => navigator.SendBeacon("/collect", new Byte[30 * 1024])).ConfigureAwait(false); + + Assert.AreEqual(4, loader.Requests.Count); + } + + [Test] + public async Task SendBeaconRejectsPayloadsLargerThanTheQuota() + { + var loader = new TestDocumentLoader(); + var navigator = await OpenNavigatorAsync(loader, "https://beacon.example/").ConfigureAwait(false); + + Assert.IsFalse(navigator.SendBeacon("/collect", new Byte[BeaconOptions.DefaultQuota + 1])); + Assert.IsTrue(navigator.SendBeacon("/collect", new Byte[BeaconOptions.DefaultQuota])); + } + + [Test] + public async Task SendBeaconUsesTheConfiguredQuota() + { + var loader = new TestDocumentLoader(); + var configuration = CreateConfiguration(loader).WithBeacon(new BeaconOptions { Quota = 10 }); + var navigator = await OpenNavigatorAsync(configuration, "https://beacon.example/").ConfigureAwait(false); + + Assert.IsFalse(navigator.SendBeacon("/collect", "eleven char")); + Assert.IsTrue(navigator.SendBeacon("/collect", "ten chars!")); + } + + [Test] + public async Task SendBeaconTracksTheQuotaPerBrowsingContext() + { + var loader = new TestDocumentLoader { Complete = false }; + var configuration = CreateConfiguration(loader); + var first = await OpenNavigatorAsync(configuration, "https://beacon.example/").ConfigureAwait(false); + var second = await OpenNavigatorAsync(configuration, "https://beacon.example/").ConfigureAwait(false); + + Assert.IsTrue(first.SendBeacon("/collect", new Byte[BeaconOptions.DefaultQuota])); + Assert.IsFalse(first.SendBeacon("/collect", "x")); + Assert.IsTrue(second.SendBeacon("/collect", "x")); + } + + [Test] + public async Task SendBeaconReleasesTheQuotaWhenTheRequestFails() + { + var loader = new TestDocumentLoader { Complete = false }; + var navigator = await OpenNavigatorAsync(loader, "https://beacon.example/").ConfigureAwait(false); + + Assert.IsTrue(navigator.SendBeacon("/collect", new Byte[BeaconOptions.DefaultQuota])); + loader.Requests[0].Fail(new InvalidOperationException("offline")); + + await WaitForAsync(() => navigator.SendBeacon("/collect", "x")).ConfigureAwait(false); + } + + [Test] + public async Task SendBeaconTreatsAThrowingLoaderAsNetworkError() + { + var loader = new TestDocumentLoader { Throw = true }; + var navigator = await OpenNavigatorAsync(loader, "https://beacon.example/").ConfigureAwait(false); + + Assert.IsTrue(navigator.SendBeacon("/collect", new Byte[BeaconOptions.DefaultQuota])); + + loader.Throw = false; + await WaitForAsync(() => navigator.SendBeacon("/collect", new Byte[BeaconOptions.DefaultQuota])).ConfigureAwait(false); + } + + [Test] + public async Task SendBeaconQueuesBeyondTheMaximumOfConcurrentRequests() + { + var loader = new TestDocumentLoader { Complete = false }; + var configuration = CreateConfiguration(loader).WithBeacon(new BeaconOptions { MaxConcurrentRequests = 2 }); + var navigator = await OpenNavigatorAsync(configuration, "https://beacon.example/").ConfigureAwait(false); + + Assert.IsTrue(navigator.SendBeacon("/one")); + Assert.IsTrue(navigator.SendBeacon("/two")); + Assert.IsTrue(navigator.SendBeacon("/three")); + Assert.IsTrue(navigator.SendBeacon("/four")); + CollectionAssert.AreEqual(new[] { "/one", "/two" }, loader.Requests.Select(m => m.Target.PathName)); + + loader.Requests[1].Finish(); + await WaitForAsync(() => loader.Requests.Count == 3).ConfigureAwait(false); + loader.Requests[0].Fail(new InvalidOperationException("offline")); + await WaitForAsync(() => loader.Requests.Count == 4).ConfigureAwait(false); + + CollectionAssert.AreEqual(new[] { "/one", "/two", "/three", "/four" }, loader.Requests.Select(m => m.Target.PathName)); + } + + [Test] + public async Task QueuedBeaconsCountTowardsTheQuota() + { + var loader = new TestDocumentLoader { Complete = false }; + var configuration = CreateConfiguration(loader).WithBeacon(new BeaconOptions { MaxConcurrentRequests = 1, Quota = 10 }); + var navigator = await OpenNavigatorAsync(configuration, "https://beacon.example/").ConfigureAwait(false); + + Assert.IsTrue(navigator.SendBeacon("/collect", "12345")); + Assert.IsTrue(navigator.SendBeacon("/collect", "12345")); + Assert.IsFalse(navigator.SendBeacon("/collect", "1")); + Assert.AreEqual(1, loader.Requests.Count); + + loader.Requests[0].Finish(); + await WaitForAsync(() => loader.Requests.Count == 2).ConfigureAwait(false); + + Assert.IsTrue(navigator.SendBeacon("/collect", "12345")); + Assert.IsFalse(navigator.SendBeacon("/collect", "1")); + } + + [TestCase("text/plain", false)] + [TestCase("TEXT/PLAIN; charset=utf-8", false)] + [TestCase("application/x-www-form-urlencoded", false)] + [TestCase("multipart/form-data; boundary=x", false)] + [TestCase("application/json", true)] + [TestCase("text/csv", true)] + public async Task CrossOriginBeaconsWithOtherContentTypesArePreflighted(String contentType, Boolean preflight) + { + var server = new CorsServer(); + var navigator = await server.OpenNavigatorAsync("https://beacon.example/").ConfigureAwait(false); + var blob = new InputFile("data", contentType, Encoding.UTF8.GetBytes("{}")); + + Assert.IsTrue(navigator.SendBeacon("https://api.example/collect", blob)); + await WaitForAsync(() => server.Requests.Any(m => m.Method == HttpMethod.Post)).ConfigureAwait(false); + + Assert.AreEqual(preflight, server.Requests.Any(m => m.Method == HttpMethod.Options)); + Assert.AreEqual(contentType, server.Requests.Last().Headers[HeaderNames.ContentType]); + } + + [Test] + public async Task SameOriginBeaconsAreNotPreflighted() + { + var server = new CorsServer(); + var navigator = await server.OpenNavigatorAsync("https://beacon.example/").ConfigureAwait(false); + var blob = new InputFile("data.json", "application/json", Encoding.UTF8.GetBytes("{}")); + + Assert.IsTrue(navigator.SendBeacon("/collect", blob)); + await WaitForAsync(() => server.Requests.Count == 1).ConfigureAwait(false); + + Assert.AreEqual(HttpMethod.Post, server.Requests[0].Method); + Assert.AreEqual("https://beacon.example", server.Requests[0].Headers[HeaderNames.Origin]); + } + + [Test] + public async Task PreflightIsSentWithoutCredentialsAndTheBeaconWithThem() + { + var server = new CorsServer(); + var context = server.CreateContext(); + await context.OpenAsync(res => res.Address("https://api.example/").Content("").Cookie("id=1; Path=/")).ConfigureAwait(false); + var document = await context.OpenAsync(res => res.Address("https://beacon.example/").Content("")).ConfigureAwait(false); + var blob = new InputFile("data.json", "application/json", Encoding.UTF8.GetBytes("{\"a\":1}")); + + Assert.IsTrue(document.DefaultView.Navigator.SendBeacon("https://api.example/collect", blob)); + await WaitForAsync(() => server.Requests.Count == 2).ConfigureAwait(false); + + var preflight = server.Requests[0]; + var beacon = server.Requests[1]; + Assert.AreEqual(HttpMethod.Options, preflight.Method); + Assert.AreEqual("https://api.example/collect", preflight.Address.Href); + Assert.AreEqual("https://beacon.example", preflight.Headers[HeaderNames.Origin]); + Assert.AreEqual("POST", preflight.Headers["Access-Control-Request-Method"]); + Assert.AreEqual("content-type", preflight.Headers["Access-Control-Request-Headers"]); + Assert.IsFalse(preflight.Headers.ContainsKey(HeaderNames.Cookie)); + Assert.AreEqual(HttpMethod.Post, beacon.Method); + Assert.AreEqual("id=1", beacon.Headers[HeaderNames.Cookie]); + Assert.AreEqual("{\"a\":1}", beacon.Body); + } + + [TestCase(200, null, "true", "content-type")] + [TestCase(200, "*", "true", "content-type")] + [TestCase(200, "https://other.example", "true", "content-type")] + [TestCase(200, "https://beacon.example", null, "content-type")] + [TestCase(200, "https://beacon.example", "TRUE", "content-type")] + [TestCase(200, "https://beacon.example", "true", null)] + [TestCase(200, "https://beacon.example", "true", "*")] + [TestCase(200, "https://beacon.example", "true", "x-custom")] + [TestCase(403, "https://beacon.example", "true", "content-type")] + public async Task FailingPreflightDropsTheBeacon(Int32 status, String allowOrigin, String allowCredentials, String allowHeaders) + { + var server = new CorsServer + { + PreflightStatus = status, + AllowOrigin = allowOrigin, + AllowCredentials = allowCredentials, + AllowHeaders = allowHeaders, + }; + var navigator = await server.OpenNavigatorAsync("https://beacon.example/", new BeaconOptions { Quota = 10 }).ConfigureAwait(false); + var blob = new InputFile("data.json", "application/json", Encoding.UTF8.GetBytes("0123456789")); + + Assert.IsTrue(navigator.SendBeacon("https://api.example/collect", blob)); + await WaitForAsync(() => navigator.SendBeacon("https://api.example/collect", "0123456789")).ConfigureAwait(false); + await WaitForAsync(() => server.Requests.Count == 2).ConfigureAwait(false); + + Assert.AreEqual(HttpMethod.Options, server.Requests[0].Method); + Assert.AreEqual(HttpMethod.Post, server.Requests[1].Method); + Assert.AreEqual("0123456789", server.Requests[1].Body); + Assert.AreEqual("text/plain;charset=UTF-8", server.Requests[1].Headers[HeaderNames.ContentType]); + } + + [Test] + public async Task SuccessfulPreflightAcceptsAllowedHeadersInAList() + { + var server = new CorsServer { AllowHeaders = "X-Custom, Content-Type" }; + var navigator = await server.OpenNavigatorAsync("https://beacon.example/").ConfigureAwait(false); + var blob = new InputFile("data.json", "application/json", Encoding.UTF8.GetBytes("{}")); + + Assert.IsTrue(navigator.SendBeacon("https://api.example/collect", blob)); + await WaitForAsync(() => server.Requests.Count == 2).ConfigureAwait(false); + + Assert.AreEqual(HttpMethod.Post, server.Requests[1].Method); + } + + [TestCase("navigate")] + [TestCase("close")] + [TestCase("dispose")] + public async Task BeaconsKeepGoingWhenTheirDocumentGoesAway(String action) + { + var requester = new BlockingRequester(); + var context = BrowsingContext.New(Configuration.Default.WithNavigator().With(requester).WithDefaultLoader()); + var document = await context.OpenAsync(res => res.Address("https://beacon.example/").Content("")).ConfigureAwait(false); + + Assert.IsTrue(document.DefaultView.Navigator.SendBeacon("/collect", new Byte[BeaconOptions.DefaultQuota])); + await requester.Started.Task.ConfigureAwait(false); + + if (action == "navigate") + { + await context.OpenAsync(res => res.Address("https://beacon.example/next").Content("")).ConfigureAwait(false); + } + else if (action == "close") + { + document.Close(); + document.Dispose(); + } + else + { + context.Dispose(); + } + + await Task.Delay(50).ConfigureAwait(false); + Assert.IsFalse(requester.Token.IsCancellationRequested); + + requester.Release(); + + if (action == "navigate") + { + var navigator = context.Active.DefaultView.Navigator; + await WaitForAsync(() => navigator.SendBeacon("/collect", new Byte[BeaconOptions.DefaultQuota])).ConfigureAwait(false); + } + } + + [Test] + public async Task SendBeaconReturnsFalseWithoutLoader() + { + var context = BrowsingContext.New(Configuration.Default.WithNavigator()); + await context.OpenAsync(res => res.Address("https://beacon.example/").Content("")).ConfigureAwait(false); + + Assert.IsFalse(context.Active.DefaultView.Navigator.SendBeacon("/collect", "x")); + } + + [Test] + public async Task SendBeaconGoesThroughTheRequesterPipelineWithCookies() + { + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var configuration = Configuration.Default + .WithNavigator() + .WithCookies() + .WithVirtualRequester(request => + { + received.TrySetResult(request); + return VirtualResponse.Create(res => res.Address(request.Address).Status(204)); + }); + var context = BrowsingContext.New(configuration); + var document = await context.OpenAsync(res => res.Address("https://beacon.example/").Content("").Cookie("session=abc")).ConfigureAwait(false); + + Assert.IsTrue(document.DefaultView.Navigator.SendBeacon("/collect", "bye")); + + var completed = await Task.WhenAny(received.Task, Task.Delay(TimeSpan.FromSeconds(5))).ConfigureAwait(false); + Assert.AreSame(received.Task, completed); + var sent = await received.Task.ConfigureAwait(false); + Assert.AreEqual(HttpMethod.Post, sent.Method); + Assert.AreEqual("https://beacon.example/collect", sent.Address.Href); + Assert.AreEqual("text/plain;charset=UTF-8", sent.Headers[HeaderNames.ContentType]); + Assert.AreEqual("session=abc", sent.Headers[HeaderNames.Cookie]); + + using (var reader = new StreamReader(sent.Content)) + { + Assert.AreEqual("bye", reader.ReadToEnd()); + } + } + + private static async Task WaitForAsync(Func condition) + { + for (var i = 0; i < 100; i++) + { + if (condition.Invoke()) + { + return; + } + + await Task.Delay(10).ConfigureAwait(false); + } + + Assert.Fail("The condition was not met in time."); + } + + private static IConfiguration CreateConfiguration(TestDocumentLoader loader) => + Configuration.Default.WithNavigator().WithOnly(_ => loader); + + private static Task OpenNavigatorAsync(TestDocumentLoader loader, String address) => + OpenNavigatorAsync(CreateConfiguration(loader), address); + + private static async Task OpenNavigatorAsync(IConfiguration configuration, String address) + { + var context = BrowsingContext.New(configuration); + var document = await context.OpenAsync(res => res.Address(address).Content("beacon")).ConfigureAwait(false); + var navigator = document.DefaultView.Navigator; + Assert.IsNotNull(navigator); + return navigator; + } + + private sealed class CorsServer + { + public Int32 PreflightStatus { get; set; } = 204; + + public String AllowOrigin { get; set; } = "https://beacon.example"; + + public String AllowCredentials { get; set; } = "true"; + + public String AllowHeaders { get; set; } = "content-type"; + + public List Requests { get; } = new List(); + + public IBrowsingContext CreateContext(BeaconOptions options = null) => + BrowsingContext.New(Configuration.Default + .WithNavigator() + .WithCookies() + .WithVirtualRequester(Handle) + .WithBeacon(options)); + + public async Task OpenNavigatorAsync(String address, BeaconOptions options = null) + { + var context = CreateContext(options); + var document = await context.OpenAsync(res => res.Address(address).Content("")).ConfigureAwait(false); + return document.DefaultView.Navigator; + } + + private IResponse Handle(Request request) + { + lock (Requests) + { + Requests.Add(new ReceivedRequest(request)); + } + + if (request.Method == HttpMethod.Options) + { + return VirtualResponse.Create(res => + { + res.Address(request.Address).Status(PreflightStatus); + + if (AllowOrigin is not null) + { + res.Header(HeaderNames.AccessControlAllowOrigin, AllowOrigin); + } + + if (AllowCredentials is not null) + { + res.Header("Access-Control-Allow-Credentials", AllowCredentials); + } + + if (AllowHeaders is not null) + { + res.Header("Access-Control-Allow-Headers", AllowHeaders); + } + }); + } + + return VirtualResponse.Create(res => res.Address(request.Address).Status(204)); + } + } + + private sealed class ReceivedRequest + { + public ReceivedRequest(Request request) + { + Method = request.Method; + Address = request.Address; + Headers = new Dictionary(request.Headers, StringComparer.OrdinalIgnoreCase); + + using (var reader = new StreamReader(request.Content ?? Stream.Null)) + { + Body = reader.ReadToEnd(); + } + } + + public HttpMethod Method { get; } + + public Url Address { get; } + + public Dictionary Headers { get; } + + public String Body { get; } + } + + private sealed class BlockingRequester : BaseRequester + { + private readonly TaskCompletionSource _release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource Started { get; } = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + public CancellationToken Token { get; private set; } + + public void Release() => _release.TrySetResult(true); + + public override Boolean SupportsProtocol(String protocol) => true; + + protected override async Task PerformRequestAsync(Request request, CancellationToken cancel) + { + Token = cancel; + Started.TrySetResult(true); + await _release.Task.ConfigureAwait(false); + return VirtualResponse.Create(res => res.Address(request.Address).Status(204)); + } + } + + private sealed class UnserializableData + { + public override String ToString() => throw new NotSupportedException(); + } + + private sealed class TestDocumentLoader : IDocumentLoader + { + public Boolean Complete { get; set; } = true; + + public Boolean Throw { get; set; } + + public List Requests { get; } = new List(); + + public IDownload FetchAsync(DocumentRequest request) + { + if (Throw) + { + throw new InvalidOperationException("broken loader"); + } + + var beacon = new SentBeacon(request); + Requests.Add(beacon); + + if (Complete) + { + beacon.Finish(); + } + + return beacon; + } + + public IEnumerable GetDownloads() => Requests; + } + + private sealed class SentBeacon : IDownload + { + private readonly TaskCompletionSource _completion = new TaskCompletionSource(); + + public SentBeacon(DocumentRequest request) + { + Target = request.Target; + Method = request.Method; + Referer = request.Referer; + Origin = request.Headers.TryGetValue(HeaderNames.Origin, out var origin) ? origin : null; + ContentType = request.Headers.TryGetValue(HeaderNames.ContentType, out var type) ? type : null; + + using (var buffer = new MemoryStream()) + { + request.Body.CopyTo(buffer); + Body = buffer.ToArray(); + } + } + + public Url Target { get; } + + public HttpMethod Method { get; } + + public String Referer { get; } + + public String Origin { get; } + + public String ContentType { get; } + + public Byte[] Body { get; } + + public Object Source => null; + + public Task Task => _completion.Task; + + public Boolean IsCompleted => _completion.Task.IsCompleted; + + public Boolean IsRunning => !_completion.Task.IsCompleted; + + public void Finish() => _completion.TrySetResult(VirtualResponse.Create(res => res.Address(Target).Status(204))); + + public void Fail(Exception error) => _completion.TrySetException(error); + + public void Cancel() => _completion.TrySetCanceled(); + } + } +} diff --git a/src/AngleSharp.Io.Tests/IoConfigurationExtensionsTests.cs b/src/AngleSharp.Io.Tests/IoConfigurationExtensionsTests.cs index 71b4a32..873df14 100644 --- a/src/AngleSharp.Io.Tests/IoConfigurationExtensionsTests.cs +++ b/src/AngleSharp.Io.Tests/IoConfigurationExtensionsTests.cs @@ -5,7 +5,10 @@ namespace AngleSharp.Io.Tests using AngleSharp.Io.Cookie; using AngleSharp.Io.Dom; using AngleSharp.Io.IndexedDb; + using AngleSharp.Io.Reporting; + using AngleSharp.Io.ServiceWorkerApi; using AngleSharp.Io.Storage; + using AngleSharp.Io.StorageManagerApi; using NUnit.Framework; using System; using System.Linq; @@ -25,6 +28,8 @@ public void WithIoRegistersCoreServices() StorageFactory = StorageProviderFactory.CreateTemporary(), IndexedDbFactory = IndexedDbProviderFactory.CreateTemporary(), CacheFactory = CacheProviderFactory.CreateTemporary(), + Beacon = new BeaconOptions { Quota = 1024 }, + Reporting = new ReportingOptions(), }; var configuration = Configuration.Default.WithIo(options); @@ -34,6 +39,9 @@ public void WithIoRegistersCoreServices() Assert.AreSame(options.StorageFactory, configuration.Services.OfType().FirstOrDefault()); Assert.AreSame(options.IndexedDbFactory, configuration.Services.OfType().FirstOrDefault()); Assert.AreSame(options.CacheFactory, configuration.Services.OfType().FirstOrDefault()); + Assert.AreSame(options.Beacon, configuration.Services.OfType().FirstOrDefault()); + Assert.AreSame(options.Reporting, configuration.Services.OfType().Single().Options); + Assert.IsTrue(requesters.Any(m => m is ReportingRequester)); Assert.IsTrue(requesters.Any(m => m.SupportsProtocol("http"))); Assert.IsTrue(requesters.Any(m => m.SupportsProtocol("data"))); Assert.IsTrue(requesters.Any(m => m.SupportsProtocol("ftp"))); @@ -52,6 +60,7 @@ public void WithIoAcceptsNullOptions() Assert.IsNotNull(configuration.Services.OfType().FirstOrDefault()); Assert.IsNotNull(configuration.Services.OfType().FirstOrDefault()); Assert.IsNotNull(configuration.Services.OfType().FirstOrDefault()); + Assert.IsNull(configuration.Services.OfType().FirstOrDefault()); } [Test] @@ -61,6 +70,8 @@ public async Task WithIoRegistersOptionalNavigatorApisWhenPlatformsAreProvided() { ClipboardPlatform = new FakeClipboardPlatform(), GeolocationPlatform = new FakeGeolocationPlatform(), + StorageManagerPlatform = new MemoryStorageManagerPlatform(), + ServiceWorkerPlatform = new ServiceWorkerPlatform(), }; var context = await OpenContextAsync(Configuration.Default.WithIo(options), "https://io.example/").ConfigureAwait(false); @@ -68,6 +79,8 @@ public async Task WithIoRegistersOptionalNavigatorApisWhenPlatformsAreProvided() Assert.IsNotNull(navigator.Clipboard()); Assert.IsNotNull(navigator.Geolocation()); + Assert.IsNotNull(navigator.Storage()); + Assert.IsNotNull(navigator.ServiceWorker()); } [Test] @@ -78,6 +91,8 @@ public async Task WithIoLeavesOptionalNavigatorApisUnavailableByDefault() Assert.IsNull(navigator.Clipboard()); Assert.IsNull(navigator.Geolocation()); + Assert.IsNull(navigator.Storage()); + Assert.IsNull(navigator.ServiceWorker()); } private static async Task OpenContextAsync(IConfiguration configuration, String address) diff --git a/src/AngleSharp.Io.Tests/Network/XmlHttpRequesterTests.cs b/src/AngleSharp.Io.Tests/Network/XmlHttpRequesterTests.cs index 09329f3..a1b7c1e 100644 --- a/src/AngleSharp.Io.Tests/Network/XmlHttpRequesterTests.cs +++ b/src/AngleSharp.Io.Tests/Network/XmlHttpRequesterTests.cs @@ -230,11 +230,9 @@ public void SerializeFormDataSetBodyShouldProduceMultipartContent() formData.Append("name", "Ada Lovelace", "text/plain"); formData.Append("role", "Mathematician", "text/plain"); - var method = typeof(XmlHttpRequest).GetMethod("Serialize", BindingFlags.Static | BindingFlags.NonPublic); - var serialized = method.Invoke(null, new Object[] { formData }); - var serializedType = serialized.GetType(); - var content = (Stream)serializedType.GetProperty("Content").GetValue(serialized); - var contentType = (String)serializedType.GetProperty("ContentType").GetValue(serialized); + var serialized = RequestBody.Serialize(formData); + var content = serialized.Content; + var contentType = serialized.ContentType; String body; @@ -254,11 +252,9 @@ public void SerializeFormDataSetBodyShouldProduceMultipartContent() public void SerializeUrlSearchParamsShouldUseWwwFormUrlEncodedContentType() { var searchParams = new UrlSearchParams("a=1&b=2"); - var method = typeof(XmlHttpRequest).GetMethod("Serialize", BindingFlags.Static | BindingFlags.NonPublic); - var serialized = method.Invoke(null, new Object[] { searchParams }); - var serializedType = serialized.GetType(); - var content = (Stream)serializedType.GetProperty("Content").GetValue(serialized); - var contentType = (String)serializedType.GetProperty("ContentType").GetValue(serialized); + var serialized = RequestBody.Serialize(searchParams); + var content = serialized.Content; + var contentType = serialized.ContentType; String body; @@ -274,11 +270,9 @@ public void SerializeUrlSearchParamsShouldUseWwwFormUrlEncodedContentType() [Test] public void SerializeNullShouldProduceEmptyBodyWithoutContentType() { - var method = typeof(XmlHttpRequest).GetMethod("Serialize", BindingFlags.Static | BindingFlags.NonPublic); - var serialized = method.Invoke(null, new Object[] { null }); - var serializedType = serialized.GetType(); - var content = (Stream)serializedType.GetProperty("Content").GetValue(serialized); - var contentType = (String)serializedType.GetProperty("ContentType").GetValue(serialized); + var serialized = RequestBody.Serialize(null); + var content = serialized.Content; + var contentType = serialized.ContentType; Assert.AreEqual(String.Empty, new StreamReader(content, Encoding.UTF8, true, 1024, true).ReadToEnd()); Assert.IsNull(contentType); diff --git a/src/AngleSharp.Io.Tests/Permissions/PermissionsTests.cs b/src/AngleSharp.Io.Tests/Permissions/PermissionsTests.cs new file mode 100644 index 0000000..55747f6 --- /dev/null +++ b/src/AngleSharp.Io.Tests/Permissions/PermissionsTests.cs @@ -0,0 +1,354 @@ +namespace AngleSharp.Io.Tests.Permissions +{ + using AngleSharp; + using AngleSharp.Browser.Dom; + using AngleSharp.Dom; + using AngleSharp.Io.Dom; + using AngleSharp.Io.PermissionsApi; + using AngleSharp.Io.StorageManagerApi; + using NUnit.Framework; + using System; + using System.Threading.Tasks; + + [TestFixture] + public class PermissionsTests + { + private const String Address = "https://permissions.example/page"; + private const String Origin = "https://permissions.example"; + + [Test] + public async Task PermissionsAreUnavailableWithoutPolicy() + { + var context = await OpenContextAsync(Configuration.Default.WithNavigator()).ConfigureAwait(false); + + Assert.IsNull(GetNavigator(context).Permissions()); + } + + [Test] + public async Task PermissionsAreUnavailableWhenRegisteredWithoutPolicy() + { + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithPermissions(null)).ConfigureAwait(false); + + Assert.IsNull(GetNavigator(context).Permissions()); + } + + [Test] + public async Task PermissionsAreTheSameInstanceForTheSameNavigator() + { + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithPermissions()).ConfigureAwait(false); + var navigator = GetNavigator(context); + + Assert.IsNotNull(navigator.Permissions()); + Assert.AreSame(navigator.Permissions(), navigator.Permissions()); + } + + [Test] + public async Task PermissionsCanBeRegisteredViaIoOptions() + { + var policy = new PermissionPolicy(PermissionState.Granted); + var context = await OpenContextAsync(Configuration.Default.WithIo(new IoOptions { Permissions = policy })).ConfigureAwait(false); + + Assert.AreSame(policy, GetNavigator(context).Permissions().Policy); + } + + [Test] + public async Task QueryReturnsTheDefaultState() + { + var permissions = await GetPermissionsAsync(new PermissionPolicy()).ConfigureAwait(false); + + var status = await permissions.Query(Describe(PermissionNames.Geolocation)).ConfigureAwait(false); + + Assert.AreEqual(PermissionNames.Geolocation, status.Name); + Assert.AreEqual(PermissionState.Prompt, status.State); + } + + [Test] + public async Task QueryPrefersTheStateOfTheOriginOverTheStateOfAllOrigins() + { + var policy = new PermissionPolicy(); + policy.Set(PermissionNames.Notifications, PermissionState.Denied); + policy.Set(Origin, PermissionNames.Notifications, PermissionState.Granted); + policy.Set("https://other.example", PermissionNames.ClipboardRead, PermissionState.Granted); + var permissions = await GetPermissionsAsync(policy).ConfigureAwait(false); + + var notifications = await permissions.Query(Describe(PermissionNames.Notifications)).ConfigureAwait(false); + var clipboard = await permissions.Query(Describe(PermissionNames.ClipboardRead)).ConfigureAwait(false); + + Assert.AreEqual(PermissionState.Granted, notifications.State); + Assert.AreEqual(PermissionState.Prompt, clipboard.State); + } + + [Test] + public void PolicyNormalizesTheOrigin() + { + var policy = new PermissionPolicy(); + + policy.Set(Origin + "/some/path?query", PermissionNames.Geolocation, PermissionState.Granted); + + Assert.AreEqual(PermissionState.Granted, policy.Query(Origin, PermissionNames.Geolocation)); + Assert.AreEqual(PermissionState.Granted, policy.Query(Origin + "/", PermissionNames.Geolocation)); + } + + [Test] + public async Task QueryRejectsUnknownNames() + { + var permissions = await GetPermissionsAsync(new PermissionPolicy()).ConfigureAwait(false); + + Assert.ThrowsAsync(() => permissions.Query(Describe("teleportation"))); + Assert.ThrowsAsync(() => permissions.Query(Describe(null))); + Assert.ThrowsAsync(() => permissions.Query(null)); + } + + [Test] + public async Task QueryAcceptsRegisteredNames() + { + var policy = new PermissionPolicy(PermissionState.Denied); + policy.Register("teleportation"); + var permissions = await GetPermissionsAsync(policy).ConfigureAwait(false); + + var status = await permissions.Query(Describe("teleportation")).ConfigureAwait(false); + + Assert.AreEqual(PermissionState.Denied, status.State); + } + + [Test] + public void SettingAnUnknownNameThrows() + { + var policy = new PermissionPolicy(); + + Assert.Throws(() => policy.Set("teleportation", PermissionState.Granted)); + Assert.Throws(() => policy.Set(Origin, "teleportation", PermissionState.Granted)); + Assert.Throws(() => policy.Set(String.Empty, PermissionNames.Geolocation, PermissionState.Granted)); + } + + [Test] + public async Task StatusFollowsChangesOfThePolicy() + { + var policy = new PermissionPolicy(); + var permissions = await GetPermissionsAsync(policy).ConfigureAwait(false); + var status = await permissions.Query(Describe(PermissionNames.Geolocation)).ConfigureAwait(false); + var changes = 0; + status.Changed += (s, e) => changes++; + + policy.Set(Origin, PermissionNames.Geolocation, PermissionState.Granted); + + Assert.AreEqual(PermissionState.Granted, status.State); + Assert.AreEqual(1, changes); + + policy.Set(PermissionNames.Geolocation, PermissionState.Denied); + + Assert.AreEqual(PermissionState.Granted, status.State); + Assert.AreEqual(1, changes); + + policy.Set(Origin, PermissionNames.Geolocation, PermissionState.Denied); + + Assert.AreEqual(PermissionState.Denied, status.State); + Assert.AreEqual(2, changes); + } + + [Test] + public async Task StatusIgnoresChangesOfOtherOriginsAndPermissions() + { + var policy = new PermissionPolicy(); + var permissions = await GetPermissionsAsync(policy).ConfigureAwait(false); + var status = await permissions.Query(Describe(PermissionNames.Geolocation)).ConfigureAwait(false); + var changes = 0; + status.Changed += (s, e) => changes++; + + policy.Set("https://other.example", PermissionNames.Geolocation, PermissionState.Granted); + policy.Set(Origin, PermissionNames.Notifications, PermissionState.Granted); + policy.Set(Origin, PermissionNames.Geolocation, PermissionState.Prompt); + + Assert.AreEqual(PermissionState.Prompt, status.State); + Assert.AreEqual(0, changes); + } + + [Test] + public async Task PromptWithoutHandlerIsDismissed() + { + var policy = new PermissionPolicy(); + var permissions = await GetPermissionsAsync(policy).ConfigureAwait(false); + + var state = await permissions.RequestAsync(PermissionNames.Geolocation).ConfigureAwait(false); + + Assert.AreEqual(PermissionState.Denied, state); + Assert.AreEqual(PermissionState.Prompt, permissions.GetState(PermissionNames.Geolocation)); + } + + [Test] + public async Task PromptHandlerAnswersAreRememberedForTheOrigin() + { + var prompts = 0; + var policy = new PermissionPolicy + { + PromptHandler = (origin, name) => + { + prompts++; + Assert.AreEqual(Origin, origin); + return Task.FromResult(PermissionState.Granted); + }, + }; + var permissions = await GetPermissionsAsync(policy).ConfigureAwait(false); + var status = await permissions.Query(Describe(PermissionNames.Geolocation)).ConfigureAwait(false); + var changes = 0; + status.Changed += (s, e) => changes++; + + await permissions.RequestAsync(PermissionNames.Geolocation).ConfigureAwait(false); + await permissions.RequestAsync(PermissionNames.Geolocation).ConfigureAwait(false); + + Assert.AreEqual(1, prompts); + Assert.AreEqual(1, changes); + Assert.AreEqual(PermissionState.Granted, status.State); + } + + [Test] + public async Task PromptHandlerLeavingThePromptDeniesWithoutRemembering() + { + var policy = new PermissionPolicy + { + PromptHandler = (origin, name) => Task.FromResult(PermissionState.Prompt), + }; + var permissions = await GetPermissionsAsync(policy).ConfigureAwait(false); + + var state = await permissions.RequestAsync(PermissionNames.Geolocation).ConfigureAwait(false); + + Assert.AreEqual(PermissionState.Denied, state); + Assert.AreEqual(PermissionState.Prompt, permissions.GetState(PermissionNames.Geolocation)); + } + + [Test] + public async Task ClipboardIsDeniedWithoutPermission() + { + var platform = new FakeClipboardPlatform(); + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithPermissions().WithClipboard(platform)).ConfigureAwait(false); + var clipboard = GetNavigator(context).Clipboard(); + + var error = Assert.ThrowsAsync(() => clipboard.WriteText("copied")); + + Assert.AreEqual("NotAllowedError", error.Name); + Assert.IsNull(platform.LastWrittenText); + } + + [Test] + public async Task ClipboardChecksReadAndWriteSeparately() + { + var policy = new PermissionPolicy(PermissionState.Denied); + policy.Set(PermissionNames.ClipboardWrite, PermissionState.Granted); + var platform = new FakeClipboardPlatform(); + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithPermissions(policy).WithClipboard(platform)).ConfigureAwait(false); + var clipboard = GetNavigator(context).Clipboard(); + + await clipboard.WriteText("copied").ConfigureAwait(false); + + Assert.AreEqual("copied", platform.LastWrittenText); + Assert.ThrowsAsync(() => clipboard.ReadText()); + } + + [Test] + public async Task ClipboardFollowsChangesOfThePolicy() + { + var policy = new PermissionPolicy(PermissionState.Denied); + var platform = new FakeClipboardPlatform(); + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithPermissions(policy).WithClipboard(platform)).ConfigureAwait(false); + var clipboard = GetNavigator(context).Clipboard(); + + Assert.ThrowsAsync(() => clipboard.WriteText("first")); + + policy.Set(Origin, PermissionNames.ClipboardWrite, PermissionState.Granted); + await clipboard.WriteText("second").ConfigureAwait(false); + + Assert.AreEqual("second", platform.LastWrittenText); + } + + [Test] + public async Task GeolocationIsGatedByThePermission() + { + var policy = new PermissionPolicy(PermissionState.Denied); + var platform = new FakeGeolocationPlatform(); + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithPermissions(policy).WithGeolocation(platform)).ConfigureAwait(false); + var geolocation = GetNavigator(context).Geolocation(); + + Assert.ThrowsAsync(() => geolocation.GetCurrentPosition()); + Assert.AreEqual(0, platform.Calls); + + policy.Set(Origin, PermissionNames.Geolocation, PermissionState.Granted); + var position = await geolocation.GetCurrentPosition().ConfigureAwait(false); + + Assert.AreEqual(1, platform.Calls); + Assert.AreEqual(47.0, position.Coordinates.Latitude); + } + + [Test] + public async Task PersistResolvesWithFalseWithoutPermission() + { + var platform = new MemoryStorageManagerPlatform(); + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithPermissions().WithStorageManager(platform)).ConfigureAwait(false); + var storage = GetNavigator(context).Storage(); + + var persisted = await storage.Persist().ConfigureAwait(false); + + Assert.IsFalse(persisted); + Assert.IsFalse(await storage.Persisted().ConfigureAwait(false)); + } + + [Test] + public async Task PersistUsesThePlatformWithPermission() + { + var policy = new PermissionPolicy(); + policy.Set(PermissionNames.PersistentStorage, PermissionState.Granted); + var platform = new MemoryStorageManagerPlatform(); + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithPermissions(policy).WithStorageManager(platform)).ConfigureAwait(false); + var storage = GetNavigator(context).Storage(); + + var persisted = await storage.Persist().ConfigureAwait(false); + + Assert.IsTrue(persisted); + } + + private static PermissionDescriptor Describe(String name) => new PermissionDescriptor { Name = name }; + + private static async Task GetPermissionsAsync(IPermissionPolicy policy) + { + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithPermissions(policy)).ConfigureAwait(false); + return GetNavigator(context).Permissions(); + } + + private static async Task OpenContextAsync(IConfiguration configuration) + { + var context = BrowsingContext.New(configuration); + await context.OpenAsync(res => res.Address(Address).Content("permissions")).ConfigureAwait(false); + return context; + } + + private static INavigator GetNavigator(IBrowsingContext context) + { + var navigator = context?.Active?.DefaultView?.Navigator; + Assert.IsNotNull(navigator); + return navigator; + } + + private sealed class FakeClipboardPlatform : IClipboardPlatform + { + public String LastWrittenText { get; private set; } + + public Task ReadTextAsync() => Task.FromResult(LastWrittenText ?? String.Empty); + + public Task WriteTextAsync(String text) + { + LastWrittenText = text; + return Task.CompletedTask; + } + } + + private sealed class FakeGeolocationPlatform : IGeolocationPlatform + { + public Int32 Calls { get; private set; } + + public Task GetCurrentPositionAsync(GeolocationOptions options) + { + Calls++; + return Task.FromResult(new GeolocationReading { Latitude = 47.0, Longitude = 11.0 }); + } + } + } +} diff --git a/src/AngleSharp.Io.Tests/Reporting/ReportingTests.cs b/src/AngleSharp.Io.Tests/Reporting/ReportingTests.cs new file mode 100644 index 0000000..cf0a656 --- /dev/null +++ b/src/AngleSharp.Io.Tests/Reporting/ReportingTests.cs @@ -0,0 +1,800 @@ +namespace AngleSharp.Io.Tests.Reporting +{ + using AngleSharp; + using AngleSharp.Browser; + using AngleSharp.Common; + using AngleSharp.Dom; + using AngleSharp.Io; + using AngleSharp.Io.Dom; + using AngleSharp.Io.Http; + using AngleSharp.Io.Network; + using AngleSharp.Io.Reporting; + using AngleSharp.Io.ServiceWorkerApi; + using AngleSharp.Io.Tests.Mocks; + using NUnit.Framework; + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Net.Sockets; + using System.Security.Authentication; + using System.Threading; + using System.Threading.Tasks; + using HttpMethod = AngleSharp.Io.HttpMethod; + + [TestFixture] + public class ReportingTests + { + private static readonly DateTimeOffset Start = new DateTimeOffset(2026, 10, 19, 12, 0, 0, TimeSpan.Zero); + + [Test] + public void ReportSerializesTypeUrlAndBody() + { + var report = new Report(ReportTypes.NetworkError, "https://report.example/page", new NetworkErrorReportBody + { + ElapsedTime = 42, + Phase = "connection", + Type = "tcp.refused", + Method = "GET", + Referrer = "https://report.example/", + }); + + Assert.AreEqual( + "{\"body\":{\"elapsed_time\":42,\"method\":\"GET\",\"phase\":\"connection\",\"protocol\":\"\",\"referrer\":\"https://report.example/\",\"sampling_fraction\":1,\"server_ip\":\"\",\"status_code\":0,\"type\":\"tcp.refused\"},\"type\":\"network-error\",\"url\":\"https://report.example/page\"}", + report.ToJson()); + } + + [Test] + public void ReportSerializesGenericBodies() + { + var body = new GenericReportBody(new Dictionary + { + ["message"] = "say \"hi\"\n\\", + ["line"] = 3, + ["ratio"] = 0.5, + ["flags"] = new Object[] { true, false, null }, + ["nested"] = new Dictionary { ["b"] = 2L, ["a"] = "x" }, + ["control"] = "\u0001", + }); + + var report = new Report("custom", "https://report.example/", body); + + Assert.AreEqual( + "{\"control\":\"\\u0001\",\"flags\":[true,false,null],\"line\":3,\"message\":\"say \\\"hi\\\"\\n\\\\\",\"nested\":{\"a\":\"x\",\"b\":2},\"ratio\":0.5}", + body.ToJson()); + StringAssert.StartsWith("{\"body\":{\"control\"", report.ToJson()); + } + + [Test] + public void ReportWithoutBodySerializesNull() + { + var report = new Report("custom", null, null); + + Assert.AreEqual("{\"body\":null,\"type\":\"custom\",\"url\":\"\"}", report.ToJson()); + Assert.AreEqual(Report.DefaultDestination, report.Destination); + Assert.Throws(() => new Report(null, "https://report.example/", null)); + Assert.Throws(() => new Report(String.Empty, "https://report.example/", null)); + } + + [Test] + public void DeliveryFormatContainsAgeAndUserAgent() + { + var service = CreateService(out var clock); + var report = new Report("custom", "https://report.example/", null, "main"); + service.QueueReport(report); + clock.Advance(TimeSpan.FromSeconds(1.5)); + + Assert.AreEqual(Start, report.Timestamp); + Assert.AreEqual( + "{\"age\":1500,\"body\":null,\"type\":\"custom\",\"url\":\"https://report.example/\",\"user_agent\":\"\"}", + report.ToDeliveryJson(clock.Now)); + } + + [Test] + public void QueuedReportsCanBeEnumeratedAndFiltered() + { + var service = CreateService(out _); + var a = Queue(service, "deprecation", "main"); + var b = Queue(service, ReportTypes.NetworkError, "default"); + var c = Queue(service, ReportTypes.NetworkError, "nel"); + + CollectionAssert.AreEqual(new[] { a, b, c }, service.GetReports()); + CollectionAssert.AreEqual(new[] { b, c }, service.GetReports(ReportFilter.ForTypes(ReportTypes.NetworkError))); + CollectionAssert.AreEqual(new[] { a, c }, service.GetReports(ReportFilter.ForDestinations("main", "nel"))); + + var both = ReportFilter.ForTypes(ReportTypes.NetworkError); + both.Destinations.Add("nel"); + CollectionAssert.AreEqual(new[] { c }, service.GetReports(both)); + Assert.IsFalse(both.Matches(null)); + } + + [Test] + public void TheOptionsFilterDecidesWhichReportsAreQueued() + { + var options = new ReportingOptions { Filter = ReportFilter.ForTypes("deprecation") }; + var service = new ReportingService(options); + var queued = new List(); + service.ReportQueued += (s, e) => queued.Add(e.Report); + + Assert.IsTrue(service.QueueReport(new Report("deprecation", "https://report.example/", null))); + Assert.IsFalse(service.QueueReport(new Report("intervention", "https://report.example/", null))); + + Assert.AreEqual(1, service.GetReports().Count); + CollectionAssert.AreEqual(service.GetReports(), queued); + Assert.Throws(() => service.QueueReport(null)); + } + + [Test] + public void TheQueueDropsTheOldestReports() + { + var service = new ReportingService(new ReportingOptions { MaxQueueSize = 2 }); + Queue(service, "first", null); + var second = Queue(service, "second", null); + var third = Queue(service, "third", null); + + CollectionAssert.AreEqual(new[] { second, third }, service.GetReports()); + } + + [Test] + public async Task FlushHandsTheReportsToTheMatchingSinks() + { + var all = new MemoryReportSink(); + var errors = new MemoryReportSink(); + var main = new MemoryReportSink(); + var options = new ReportingOptions() + .AddSink(all) + .AddSink(errors, ReportFilter.ForTypes(ReportTypes.NetworkError)) + .AddSink(main, ReportFilter.ForDestinations("main")); + var service = new ReportingService(options); + var a = Queue(service, "deprecation", "main"); + var b = Queue(service, ReportTypes.NetworkError, "default"); + + var flushed = await service.FlushAsync().ConfigureAwait(false); + + Assert.AreEqual(2, flushed); + CollectionAssert.AreEqual(new[] { a, b }, all.Reports); + CollectionAssert.AreEqual(new[] { b }, errors.Reports); + CollectionAssert.AreEqual(new[] { a }, main.Reports); + Assert.IsEmpty(service.GetReports()); + Assert.AreEqual(0, await service.FlushAsync().ConfigureAwait(false)); + Assert.AreEqual(2, all.Reports.Count); + + all.Clear(); + Assert.IsEmpty(all.Reports); + } + + [Test] + public async Task FlushAcceptsCallbacksAsSinks() + { + var received = new List(); + var options = new ReportingOptions().AddSink((reports, cancel) => + { + received.AddRange(reports); + return Task.CompletedTask; + }); + var service = new ReportingService(options); + var report = Queue(service, "custom", null); + + await service.FlushAsync().ConfigureAwait(false); + + CollectionAssert.AreEqual(new[] { report }, received); + Assert.Throws(() => options.AddSink((Func, CancellationToken, Task>)null)); + Assert.Throws(() => options.AddSink((IReportSink)null)); + } + + [Test] + public void FailingSinksAreRethrownAfterAllSinksWereCalled() + { + var memory = new MemoryReportSink(); + var options = new ReportingOptions() + .AddSink((reports, cancel) => Task.FromException(new InvalidOperationException("first"))) + .AddSink(memory); + var service = new ReportingService(options); + Queue(service, "custom", null); + + Assert.ThrowsAsync(() => service.FlushAsync()); + Assert.AreEqual(1, memory.Reports.Count); + Assert.IsEmpty(service.GetReports()); + + options.AddSink((reports, cancel) => throw new NotSupportedException("second")); + Queue(service, "custom", null); + + var ex = Assert.ThrowsAsync(() => service.FlushAsync()); + Assert.AreEqual(2, ex.InnerExceptions.Count); + } + + [Test] + public async Task FlushIntervalFlushesAutomatically() + { + var sink = new MemoryReportSink(); + var options = new ReportingOptions { FlushInterval = TimeSpan.FromMilliseconds(20) }.AddSink(sink); + + using (var service = new ReportingService(options)) + { + Queue(service, "custom", null); + await WaitForAsync(() => sink.Reports.Count == 1).ConfigureAwait(false); + Assert.IsEmpty(service.GetReports()); + } + } + + [Test] + public void ConfiguredEndpointsResolveTheDestination() + { + var options = new ReportingOptions(); + options.Endpoints["default"] = "https://collector.example/reports"; + var service = new ReportingService(options); + + var known = Queue(service, "custom", "default"); + var unknown = Queue(service, "custom", "other"); + + Assert.AreEqual("https://collector.example/reports", known.EndpointUrl); + Assert.IsNull(unknown.EndpointUrl); + } + + [Test] + public async Task ReportingEndpointsHeadersAnnounceEndpointsPerOrigin() + { + var options = new ReportingOptions(); + options.Endpoints["main"] = "https://fallback.example/"; + var service = new ReportingService(options); + var requester = new ReportingRequester(new VirtualRequester(request => Respond(request, 200, ("Reporting-Endpoints", "main=\"/reports\", csp=\"https://csp.example/a,b\", bad=\"ftp://x/\""))), service); + + await requester.RequestAsync(Get("https://report.example/page"), CancellationToken.None).ConfigureAwait(false); + + Assert.AreEqual("https://report.example/reports", Queue(service, "custom", "main", "https://report.example/other").EndpointUrl); + Assert.AreEqual("https://csp.example/a,b", Queue(service, "custom", "csp", "https://report.example/").EndpointUrl); + Assert.IsNull(Queue(service, "custom", "bad", "https://report.example/").EndpointUrl); + Assert.AreEqual("https://fallback.example/", Queue(service, "custom", "main", "https://elsewhere.example/").EndpointUrl); + } + + [Test] + public async Task EndpointSinkPostsReportsGroupedByEndpoint() + { + var posted = new List<(String Url, String ContentType, String Body)>(); + var transport = new VirtualRequester(request => + { + using (var reader = new StreamReader(request.Content)) + { + posted.Add((request.Address.Href, request.Headers[HeaderNames.ContentType], reader.ReadToEnd())); + } + + return Respond(request, 204); + }); + var sink = new EndpointReportSink(transport) { Clock = () => Start.AddSeconds(2) }; + var options = new ReportingOptions { Clock = () => Start }.AddSink(sink); + options.Endpoints["a"] = "https://a.example/reports"; + options.Endpoints["b"] = "https://b.example/reports"; + var service = new ReportingService(options); + Queue(service, "one", "a"); + Queue(service, "two", "b"); + Queue(service, "three", "a"); + Queue(service, "four", "unknown"); + + await service.FlushAsync().ConfigureAwait(false); + + Assert.AreEqual(2, posted.Count); + Assert.AreEqual("https://a.example/reports", posted[0].Url); + Assert.AreEqual("application/reports+json", posted[0].ContentType); + Assert.AreEqual( + "[{\"age\":2000,\"body\":null,\"type\":\"one\",\"url\":\"https://report.example/\",\"user_agent\":\"\"},{\"age\":2000,\"body\":null,\"type\":\"three\",\"url\":\"https://report.example/\",\"user_agent\":\"\"}]", + posted[0].Body); + Assert.AreEqual("https://b.example/reports", posted[1].Url); + Assert.Throws(() => new EndpointReportSink(null)); + } + + [Test] + public void EndpointSinkFailsForUnsuccessfulDeliveries() + { + var transport = new VirtualRequester(request => Respond(request, request.Address.Host == "a.example" ? 500 : 204)); + var options = new ReportingOptions().AddSink(new EndpointReportSink(transport)); + options.Endpoints["a"] = "https://a.example/reports"; + options.Endpoints["b"] = "https://b.example/reports"; + var service = new ReportingService(options); + Queue(service, "one", "a"); + Queue(service, "two", "b"); + + var ex = Assert.ThrowsAsync(() => service.FlushAsync()); + + StringAssert.Contains("https://a.example/reports", ex.Message); + StringAssert.DoesNotContain("https://b.example/reports", ex.Message); + } + + [Test] + public async Task HttpErrorsAreReportedWithTheRequestDetails() + { + var service = CreateService(out _); + var requester = new ReportingRequester(new VirtualRequester(request => Respond(request, 404)), service); + var request = Get("https://user:pass@report.example/missing?q=1#top"); + request.Headers[HeaderNames.Referer] = "https://report.example/"; + request.Headers[HeaderNames.UserAgent] = "TestAgent/1.0"; + + var response = await requester.RequestAsync(request, CancellationToken.None).ConfigureAwait(false); + + Assert.AreEqual(HttpStatusCode.NotFound, response.StatusCode); + var report = service.GetReports().Single(); + var body = (NetworkErrorReportBody)report.Body; + Assert.AreEqual(ReportTypes.NetworkError, report.Type); + Assert.AreEqual("https://report.example/missing?q=1", report.Url); + Assert.AreEqual("default", report.Destination); + Assert.AreEqual("TestAgent/1.0", report.UserAgent); + Assert.AreEqual("application", body.Phase); + Assert.AreEqual("http.error", body.Type); + Assert.AreEqual(404, body.StatusCode); + Assert.AreEqual("GET", body.Method); + Assert.AreEqual("https://report.example/", body.Referrer); + Assert.GreaterOrEqual(body.ElapsedTime, 0); + } + + [Test] + public async Task SuccessfulResponsesAreNotReported() + { + var service = CreateService(out _); + var requester = new ReportingRequester(new VirtualRequester(request => Respond(request, 302)), service); + + await requester.RequestAsync(Get("https://report.example/"), CancellationToken.None).ConfigureAwait(false); + + Assert.IsEmpty(service.GetReports()); + } + + [Test] + public async Task CapturingCanBeTurnedOff() + { + var service = new ReportingService(new ReportingOptions { CaptureHttpErrors = false, CaptureNetworkErrors = false }); + var requester = new ReportingRequester(new VirtualRequester(request => Respond(request, request.Address.PathName == "/missing" ? 404 : 0)), service); + + await requester.RequestAsync(Get("https://report.example/missing"), CancellationToken.None).ConfigureAwait(false); + await requester.RequestAsync(Get("https://report.example/failed"), CancellationToken.None).ConfigureAwait(false); + + Assert.IsEmpty(service.GetReports()); + } + + [Test] + public async Task NetworkErrorsUseTheConfiguredDestination() + { + var service = new ReportingService(new ReportingOptions { NetworkErrorDestination = "nel" }); + var requester = new ReportingRequester(new VirtualRequester(request => Respond(request, 0)), service); + + await requester.RequestAsync(Get("https://report.example/"), CancellationToken.None).ConfigureAwait(false); + + var report = service.GetReports().Single(); + Assert.AreEqual("nel", report.Destination); + Assert.AreEqual("unknown", ((NetworkErrorReportBody)report.Body).Type); + } + + [Test] + public async Task FailedResponsesAreClassifiedByTheirCause() + { + var service = CreateService(out _); + var requester = new ReportingRequester(new VirtualRequester(request => + { + var response = Respond(request, 0); + NetworkFailures.Attach(response, new HttpRequestException("failed", new SocketException((Int32)SocketError.ConnectionRefused))); + return response; + }), service); + + await requester.RequestAsync(Get("https://report.example/"), CancellationToken.None).ConfigureAwait(false); + + var body = (NetworkErrorReportBody)service.GetReports().Single().Body; + Assert.AreEqual("connection", body.Phase); + Assert.AreEqual("tcp.refused", body.Type); + Assert.AreEqual(0, body.StatusCode); + } + + [Test] + public void ThrowingRequestersAreReportedAndRethrown() + { + var service = CreateService(out _); + var requester = new ReportingRequester(new ThrowingRequester(new SocketException((Int32)SocketError.HostNotFound)), service); + + Assert.ThrowsAsync(() => requester.RequestAsync(Get("https://missing.example/"), CancellationToken.None)); + + var body = (NetworkErrorReportBody)service.GetReports().Single().Body; + Assert.AreEqual("dns", body.Phase); + Assert.AreEqual("dns.name_not_resolved", body.Type); + } + + [Test] + public void CancelledRequestsAreReportedAsAbandoned() + { + var service = CreateService(out _); + var requester = new ReportingRequester(new ThrowingRequester(new OperationCanceledException()), service); + + using (var cancel = new CancellationTokenSource()) + { + cancel.Cancel(); + Assert.CatchAsync(() => requester.RequestAsync(Get("https://report.example/"), cancel.Token)); + } + + Assert.AreEqual("abandoned", ((NetworkErrorReportBody)service.GetReports().Single().Body).Type); + } + + [Test] + public void ErrorsAreClassifiedLikeNetworkErrorLogging() + { + Assert.AreEqual(("dns", "dns.name_not_resolved"), Classify(new SocketException((Int32)SocketError.HostNotFound))); + Assert.AreEqual(("dns", "dns.failed"), Classify(new SocketException((Int32)SocketError.TryAgain))); + Assert.AreEqual(("connection", "tcp.refused"), Classify(new HttpRequestException("x", new SocketException((Int32)SocketError.ConnectionRefused)))); + Assert.AreEqual(("connection", "tcp.reset"), Classify(new SocketException((Int32)SocketError.ConnectionReset))); + Assert.AreEqual(("connection", "tcp.aborted"), Classify(new SocketException((Int32)SocketError.ConnectionAborted))); + Assert.AreEqual(("connection", "tcp.timed_out"), Classify(new SocketException((Int32)SocketError.TimedOut))); + Assert.AreEqual(("connection", "tcp.address_unreachable"), Classify(new SocketException((Int32)SocketError.HostUnreachable))); + Assert.AreEqual(("connection", "tcp.address_invalid"), Classify(new SocketException((Int32)SocketError.AddressNotAvailable))); + Assert.AreEqual(("connection", "tcp.failed"), Classify(new SocketException((Int32)SocketError.Shutdown))); + Assert.AreEqual(("connection", "tls.failed"), Classify(new HttpRequestException("x", new AuthenticationException()))); + Assert.AreEqual(("connection", "tcp.timed_out"), Classify(new TimeoutException())); + Assert.AreEqual(("connection", "tcp.timed_out"), Classify(new TaskCanceledException())); + Assert.AreEqual(("application", "unknown"), Classify(new InvalidOperationException())); + Assert.AreEqual(("application", "unknown"), Classify(null)); + } + + [Test] + public async Task RefusedConnectionsOfTheHttpClientRequesterAreReported() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + + var service = CreateService(out _); + var configuration = Configuration.Default.WithRequesters().WithReporting(service); + var requester = configuration.Services.OfType().First(m => m.SupportsProtocol("http")); + + var response = await requester.RequestAsync(Get($"http://127.0.0.1:{port}/"), CancellationToken.None).ConfigureAwait(false); + + Assert.AreEqual(0, (Int32)response.StatusCode); + var body = (NetworkErrorReportBody)service.GetReports().Single().Body; + Assert.AreEqual("connection", body.Phase); + Assert.AreEqual("tcp.refused", body.Type); + } + + [Test] + public async Task NavigationsGoThroughTheReportingPipeline() + { + var service = CreateService(out _); + var configuration = Configuration.Default + .WithVirtualRequester(request => Respond(request, request.Address.PathName == "/missing" ? 404 : 200)) + .WithReporting(service); + var context = BrowsingContext.New(configuration); + + await context.OpenAsync("https://report.example/").ConfigureAwait(false); + await context.OpenAsync("https://report.example/missing").ConfigureAwait(false); + + var report = service.GetReports().Single(); + Assert.AreEqual("https://report.example/missing", report.Url); + Assert.AreEqual("http.error", ((NetworkErrorReportBody)report.Body).Type); + } + + [Test] + public void ReportingSitsClosestToTheNetwork() + { + var network = new VirtualRequester(_ => null); + + foreach (var configuration in new[] + { + Configuration.Default.With(network).WithReporting().WithHttpCache().WithServiceWorkers(), + Configuration.Default.With(network).WithServiceWorkers().WithHttpCache().WithReporting(), + Configuration.Default.With(network).WithHttpCache().WithReporting().WithServiceWorkers(), + }) + { + var serviceWorker = (ServiceWorkerRequester)configuration.Services.OfType().Single(); + var caching = (CachingRequester)serviceWorker.Inner; + var reporting = (ReportingRequester)caching.Inner; + Assert.AreSame(network, reporting.Inner); + } + } + + [Test] + public void RegisteringReportingTwiceDoesNotNestTheRequesters() + { + var network = new VirtualRequester(_ => null); + var service = CreateService(out _); + var configuration = Configuration.Default.With(network).WithReporting().WithReporting(service); + + var requester = (ReportingRequester)configuration.Services.OfType().Single(); + + Assert.AreSame(network, requester.Inner); + Assert.AreSame(service, requester.Service); + Assert.AreSame(service, configuration.Services.OfType().Single()); + Assert.AreSame(configuration, configuration.WithReporting((ReportingOptions)null)); + Assert.Throws(() => new ReportingRequester(null, service)); + Assert.Throws(() => new ReportingRequester(network, null)); + Assert.Throws(() => new ReportingService(null)); + } + + [Test] + public async Task WindowReportsReachObserversAndTheService() + { + var service = CreateService(out _); + var window = await OpenWindowAsync(Configuration.Default.WithNavigator().WithReporting(service), "https://user:pw@report.example/page#top").ConfigureAwait(false); + var observed = new List(); + var observer = new ReportingObserver(window, (reports, o) => observed.AddRange(reports)); + observer.Observe(); + + var report = window.QueueReport(ReportTypes.Deprecation, new GenericReportBody(new Dictionary { ["id"] = "old-api" }), "main"); + + CollectionAssert.AreEqual(new[] { report }, observed); + CollectionAssert.AreEqual(new[] { report }, service.GetReports()); + Assert.AreEqual("https://report.example/page", report.Url); + Assert.AreEqual("main", report.Destination); + Assert.AreEqual(window.Navigator.UserAgent, report.UserAgent); + } + + [Test] + public async Task ObserversFilterByType() + { + var window = await OpenWindowAsync(Configuration.Default, "https://report.example/").ConfigureAwait(false); + var observed = new List(); + var observer = new ReportingObserver(window, (reports, o) => observed.AddRange(reports.Select(m => m.Type)), new ReportingObserverOptions { Types = new[] { "intervention" } }); + observer.Observe(); + + window.QueueReport("deprecation", null); + window.QueueReport("intervention", null); + + CollectionAssert.AreEqual(new[] { "intervention" }, observed); + } + + [Test] + public async Task DisconnectedObserversReceiveNothing() + { + var window = await OpenWindowAsync(Configuration.Default, "https://report.example/").ConfigureAwait(false); + var observed = 0; + var observer = new ReportingObserver(window, (reports, o) => observed += reports.Length); + observer.Observe(); + observer.Observe(); + + window.QueueReport("deprecation", null); + observer.Disconnect(); + window.QueueReport("deprecation", null); + + Assert.AreEqual(1, observed); + } + + [Test] + public async Task BufferedObserversReceiveEarlierReports() + { + var window = await OpenWindowAsync(Configuration.Default, "https://report.example/").ConfigureAwait(false); + var earlier = window.QueueReport("deprecation", null); + var buffered = new List(); + var unbuffered = new List(); + new ReportingObserver(window, (reports, o) => buffered.Add(reports), new ReportingObserverOptions { Buffered = true }).Observe(); + new ReportingObserver(window, (reports, o) => unbuffered.Add(reports)).Observe(); + + Assert.AreEqual(1, buffered.Count); + CollectionAssert.AreEqual(new[] { earlier }, buffered[0]); + Assert.IsEmpty(unbuffered); + } + + [Test] + public async Task TheReportBufferKeepsTheLatestHundredReports() + { + var window = await OpenWindowAsync(Configuration.Default, "https://report.example/").ConfigureAwait(false); + + for (var i = 0; i < 105; i++) + { + window.QueueReport("r" + i, null); + } + + var observed = new List(); + new ReportingObserver(window, (reports, o) => observed.AddRange(reports), new ReportingObserverOptions { Buffered = true }).Observe(); + + Assert.AreEqual(100, observed.Count); + Assert.AreEqual("r5", observed[0].Type); + } + + [Test] + public async Task TakeRecordsReturnsTheUndeliveredReports() + { + var window = await OpenWindowAsync(Configuration.Default, "https://report.example/").ConfigureAwait(false); + var taken = default(Report[]); + var observer = default(ReportingObserver); + observer = new ReportingObserver(window, (reports, o) => taken = o.TakeRecords()); + observer.Observe(); + + window.QueueReport("deprecation", null); + + Assert.IsEmpty(taken); + Assert.IsEmpty(observer.TakeRecords()); + Assert.Throws(() => new ReportingObserver(null, (r, o) => { })); + Assert.Throws(() => new ReportingObserver(window, null)); + } + + [Test] + public async Task ObserverCallbacksAreQueuedAsTasksOnTheEventLoop() + { + var loop = new ManualEventLoop(); + var window = await OpenWindowAsync(Configuration.Default.WithOnly(_ => loop), "https://report.example/").ConfigureAwait(false); + loop.Reset(); + var calls = new List(); + new ReportingObserver(window, (reports, o) => calls.Add(reports)).Observe(); + + var first = window.QueueReport("deprecation", null); + var second = window.QueueReport("intervention", null); + + Assert.IsEmpty(calls); + CollectionAssert.AreEqual(new[] { TaskPriority.Normal }, loop.Priorities); + + loop.RunAll(); + + Assert.AreEqual(1, calls.Count); + CollectionAssert.AreEqual(new[] { first, second }, calls[0]); + + var third = window.QueueReport("deprecation", null); + Assert.AreEqual(1, calls.Count); + + loop.RunAll(); + + Assert.AreEqual(2, calls.Count); + CollectionAssert.AreEqual(new[] { third }, calls[1]); + } + + [Test] + public async Task TakingTheRecordsBeforeTheTaskRunsSkipsTheCallback() + { + var loop = new ManualEventLoop(); + var window = await OpenWindowAsync(Configuration.Default.WithOnly(_ => loop), "https://report.example/").ConfigureAwait(false); + loop.Reset(); + var calls = 0; + var observer = new ReportingObserver(window, (reports, o) => calls++); + observer.Observe(); + + var report = window.QueueReport("deprecation", null); + var taken = observer.TakeRecords(); + loop.RunAll(); + + CollectionAssert.AreEqual(new[] { report }, taken); + Assert.AreEqual(0, calls); + } + + [Test] + public async Task BufferedReportsAreDeliveredInOneTask() + { + var loop = new ManualEventLoop(); + var window = await OpenWindowAsync(Configuration.Default.WithOnly(_ => loop), "https://report.example/").ConfigureAwait(false); + loop.Reset(); + var first = window.QueueReport("deprecation", null); + var second = window.QueueReport("deprecation", null); + var calls = new List(); + + new ReportingObserver(window, (reports, o) => calls.Add(reports), new ReportingObserverOptions { Buffered = true }).Observe(); + + Assert.IsEmpty(calls); + Assert.AreEqual(1, loop.Priorities.Count); + + loop.RunAll(); + + Assert.AreEqual(1, calls.Count); + CollectionAssert.AreEqual(new[] { first, second }, calls[0]); + } + + [Test] + public async Task NetworkErrorsAreNotVisibleToObservers() + { + var service = CreateService(out _); + var configuration = Configuration.Default + .WithVirtualRequester(request => Respond(request, 500)) + .WithReporting(service); + var context = BrowsingContext.New(configuration); + var document = await context.OpenAsync(res => res.Address("https://report.example/").Content("")).ConfigureAwait(false); + var observed = 0; + new ReportingObserver(document.DefaultView, (reports, o) => observed += reports.Length, new ReportingObserverOptions { Buffered = true }).Observe(); + + await document.DefaultView.Fetch("/api").ConfigureAwait(false); + + Assert.AreEqual(1, service.GetReports().Count); + Assert.AreEqual(0, observed); + } + + private static (String, String) Classify(Exception error) => ReportingRequester.Classify(error, CancellationToken.None); + + private static ReportingService CreateService(out TestClock clock) + { + var current = new TestClock(); + clock = current; + return new ReportingService(new ReportingOptions { Clock = () => current.Now }); + } + + private static Report Queue(ReportingService service, String type, String destination, String url = "https://report.example/") + { + var report = new Report(type, url, null, destination); + service.QueueReport(report); + return report; + } + + private static Request Get(String address) => new Request + { + Method = HttpMethod.Get, + Address = new Url(address), + Content = Stream.Null, + }; + + private static IResponse Respond(Request request, Int32 status, params (String Name, String Value)[] headers) => + VirtualResponse.Create(res => + { + res.Address(request.Address).Status(status); + + foreach (var (name, value) in headers) + { + res.Header(name, value); + } + }); + + private static async Task OpenWindowAsync(IConfiguration configuration, String address) + { + var context = BrowsingContext.New(configuration); + var document = await context.OpenAsync(res => res.Address(address).Content("")).ConfigureAwait(false); + return document.DefaultView; + } + + private static async Task WaitForAsync(Func condition) + { + for (var i = 0; i < 200; i++) + { + if (condition.Invoke()) + { + return; + } + + await Task.Delay(10).ConfigureAwait(false); + } + + Assert.Fail("The condition was not met in time."); + } + + private sealed class ManualEventLoop : IEventLoop + { + private readonly Queue> _tasks = new Queue>(); + + public List Priorities { get; } = new List(); + + public ICancellable Enqueue(Action action, TaskPriority priority) + { + Priorities.Add(priority); + _tasks.Enqueue(action); + return null; + } + + public void RunAll() + { + while (_tasks.Count > 0) + { + _tasks.Dequeue().Invoke(CancellationToken.None); + } + } + + public void Reset() + { + // runs the tasks the document queued while loading + RunAll(); + Priorities.Clear(); + } + + public void CancelAll() => _tasks.Clear(); + + public void Spin() => RunAll(); + } + + private sealed class TestClock + { + public DateTimeOffset Now { get; private set; } = Start; + + public void Advance(TimeSpan time) => Now += time; + } + + private sealed class ThrowingRequester : BaseRequester + { + private readonly Exception _error; + + public ThrowingRequester(Exception error) + { + _error = error; + } + + public override Boolean SupportsProtocol(String protocol) => true; + + protected override Task PerformRequestAsync(Request request, CancellationToken cancel) => + Task.FromException(_error); + } + } +} diff --git a/src/AngleSharp.Io.Tests/ServiceWorker/ServiceWorkerTests.cs b/src/AngleSharp.Io.Tests/ServiceWorker/ServiceWorkerTests.cs new file mode 100644 index 0000000..3eec6cd --- /dev/null +++ b/src/AngleSharp.Io.Tests/ServiceWorker/ServiceWorkerTests.cs @@ -0,0 +1,815 @@ +namespace AngleSharp.Io.Tests.ServiceWorker +{ + using AngleSharp; + using AngleSharp.Browser.Dom; + using AngleSharp.Dom; + using AngleSharp.Dom.Events; + using AngleSharp.Io.Dom; + using AngleSharp.Io.Http; + using AngleSharp.Io.ServiceWorkerApi; + using NUnit.Framework; + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading.Tasks; + + [TestFixture] + public class ServiceWorkerTests + { + [Test] + public async Task ServiceWorkerContainerCanBeResolvedFromNavigatorWhenRegistered() + { + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithServiceWorkers(), "https://sw.example/").ConfigureAwait(false); + var navigator = GetNavigator(context); + + Assert.IsNotNull(navigator.ServiceWorker()); + Assert.AreSame(navigator.ServiceWorker(), navigator.ServiceWorker()); + } + + [Test] + public async Task ServiceWorkerContainerIsUnavailableWhenNotRegistered() + { + var context = await OpenContextAsync(Configuration.Default.WithNavigator(), "https://sw.example/").ConfigureAwait(false); + + Assert.IsNull(GetNavigator(context).ServiceWorker()); + } + + [Test] + public async Task ServiceWorkerContainerIsUnavailableWhenRegisteredWithoutPlatform() + { + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithServiceWorkers((IServiceWorkerPlatform)null), "https://sw.example/").ConfigureAwait(false); + + Assert.IsNull(GetNavigator(context).ServiceWorker()); + } + + [Test] + public void ServiceWorkerTypesRequireTheirDependencies() + { + Assert.Throws(() => new ServiceWorkerRegistry(null)); + Assert.Throws(() => new ServiceWorkerProviderFactory(null)); + Assert.Throws(() => new ServiceWorkerContainer(null, () => null)); + Assert.Throws(() => new ServiceWorkerContainer(new ServiceWorkerRegistry(new ServiceWorkerPlatform()), null)); + Assert.Throws(() => new ServiceWorkerRequester(null, new ServiceWorkerRegistry(new ServiceWorkerPlatform()))); + Assert.Throws(() => new ServiceWorkerRequester(new Mocks.VirtualRequester(_ => null), null)); + Assert.IsNull(new ServiceWorkerProviderFactory(new ServiceWorkerRegistry(new ServiceWorkerPlatform())).GetServiceWorkerContainer(null)); + } + + [Test] + public async Task RegisterRunsTheLifecycleUntilActivated() + { + var steps = new List(); + var platform = new ServiceWorkerPlatform + { + Install = worker => + { + steps.Add("install:" + worker.State.ToString()); + worker.StateChanged += (s, e) => steps.Add(((ServiceWorker)s).State.ToString()); + return Task.CompletedTask; + }, + Activate = worker => + { + steps.Add("activate:" + worker.State.ToString()); + return Task.CompletedTask; + }, + }; + + var container = await OpenContainerAsync(platform, "https://sw.example/app/index.html").ConfigureAwait(false); + var registration = await container.Register("/app/sw.js").ConfigureAwait(false); + + Assert.AreEqual("https://sw.example/app/", registration.Scope); + Assert.IsNull(registration.Installing); + Assert.IsNull(registration.Waiting); + Assert.IsNotNull(registration.Active); + Assert.AreEqual("https://sw.example/app/sw.js", registration.Active.ScriptUrl); + Assert.AreEqual(ServiceWorkerState.Activated, registration.Active.State); + Assert.AreSame(registration, registration.Active.Registration); + CollectionAssert.AreEqual(new[] { "install:Installing", "Installed", "Activating", "activate:Activating", "Activated" }, steps); + } + + [Test] + public async Task RegisterRaisesUpdateFoundWithTheInstallingWorker() + { + var registry = new ServiceWorkerRegistry(new ServiceWorkerPlatform()); + var found = new List(); + var registration = await registry.RegisterAsync(new Url("https://sw.example/sw.js"), new Url("https://sw.example/")).ConfigureAwait(false); + registration.UpdateFound += (s, e) => found.Add(((ServiceWorkerRegistration)s).Installing); + + await registry.RegisterAsync(new Url("https://sw.example/sw2.js"), new Url("https://sw.example/")).ConfigureAwait(false); + + Assert.AreEqual(1, found.Count); + Assert.AreEqual("https://sw.example/sw2.js", found[0].ScriptUrl); + } + + [Test] + public async Task RegisterUsesTheGivenScope() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + + var registration = await container.Register("/sw.js", new ServiceWorkerRegistrationOptions { Scope = "/app/" }).ConfigureAwait(false); + + Assert.AreEqual("https://sw.example/app/", registration.Scope); + } + + [Test] + public async Task RegisterIgnoresFragments() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + + var registration = await container.Register("/sw.js#first", new ServiceWorkerRegistrationOptions { Scope = "/app/#top" }).ConfigureAwait(false); + + Assert.AreEqual("https://sw.example/app/", registration.Scope); + Assert.AreEqual("https://sw.example/sw.js", registration.Active.ScriptUrl); + } + + [Test] + public async Task RegisteringTheSameScriptAgainReturnsTheExistingRegistration() + { + var installs = 0; + var platform = new ServiceWorkerPlatform + { + Install = _ => + { + installs++; + return Task.CompletedTask; + }, + }; + + var container = await OpenContainerAsync(platform, "https://sw.example/").ConfigureAwait(false); + var first = await container.Register("/sw.js").ConfigureAwait(false); + var worker = first.Active; + var second = await container.Register("/sw.js").ConfigureAwait(false); + + Assert.AreSame(first, second); + Assert.AreSame(worker, second.Active); + Assert.AreEqual(1, installs); + } + + [Test] + public async Task RegisteringAnotherScriptReplacesTheActiveWorker() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + var first = await container.Register("/sw.js").ConfigureAwait(false); + var previous = first.Active; + var second = await container.Register("/sw-v2.js", new ServiceWorkerRegistrationOptions { Scope = "/" }).ConfigureAwait(false); + + Assert.AreSame(first, second); + Assert.AreEqual(ServiceWorkerState.Redundant, previous.State); + Assert.AreEqual("https://sw.example/sw-v2.js", second.Active.ScriptUrl); + Assert.AreEqual(ServiceWorkerState.Activated, second.Active.State); + } + + [Test] + public async Task FailingInstallationMakesTheWorkerRedundantAndDropsTheRegistration() + { + var platform = new ServiceWorkerPlatform + { + Install = _ => Task.FromException(new InvalidOperationException("broken")), + }; + + var container = await OpenContainerAsync(platform, "https://sw.example/").ConfigureAwait(false); + var registration = await container.Register("/sw.js").ConfigureAwait(false); + + Assert.IsNull(registration.Installing); + Assert.IsNull(registration.Waiting); + Assert.IsNull(registration.Active); + Assert.IsNull(await container.GetRegistration().ConfigureAwait(false)); + Assert.IsEmpty(await container.GetRegistrations().ConfigureAwait(false)); + } + + [Test] + public async Task FailingUpdateKeepsTheActiveWorker() + { + var platform = new ServiceWorkerPlatform(); + var container = await OpenContainerAsync(platform, "https://sw.example/").ConfigureAwait(false); + var registration = await container.Register("/sw.js").ConfigureAwait(false); + var active = registration.Active; + var failed = default(ServiceWorker); + platform.Install = worker => + { + failed = worker; + return Task.FromException(new InvalidOperationException("broken")); + }; + + await container.Register("/sw-v2.js", new ServiceWorkerRegistrationOptions { Scope = "/" }).ConfigureAwait(false); + + Assert.AreSame(active, registration.Active); + Assert.AreEqual(ServiceWorkerState.Activated, active.State); + Assert.AreEqual(ServiceWorkerState.Redundant, failed.State); + Assert.AreSame(registration, await container.GetRegistration().ConfigureAwait(false)); + } + + [Test] + public async Task FailingActivationStillActivatesTheWorker() + { + var platform = new ServiceWorkerPlatform + { + Activate = _ => Task.FromException(new InvalidOperationException("broken")), + }; + + var container = await OpenContainerAsync(platform, "https://sw.example/").ConfigureAwait(false); + var registration = await container.Register("/sw.js").ConfigureAwait(false); + + Assert.AreEqual(ServiceWorkerState.Activated, registration.Active.State); + } + + [Test] + public async Task GetRegistrationMatchesTheLongestScope() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + var root = await container.Register("/sw.js").ConfigureAwait(false); + var app = await container.Register("/app/sw.js").ConfigureAwait(false); + + Assert.AreSame(root, await container.GetRegistration().ConfigureAwait(false)); + Assert.AreSame(root, await container.GetRegistration("/other/page").ConfigureAwait(false)); + Assert.AreSame(app, await container.GetRegistration("/app/").ConfigureAwait(false)); + Assert.AreSame(app, await container.GetRegistration("https://sw.example/app/deep/page?x=1").ConfigureAwait(false)); + Assert.AreSame(root, await container.GetRegistration("/application").ConfigureAwait(false)); + } + + [Test] + public async Task GetRegistrationMatchesScopesAsPrefix() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + var registration = await container.Register("/sw.js", new ServiceWorkerRegistrationOptions { Scope = "/app" }).ConfigureAwait(false); + + Assert.AreSame(registration, await container.GetRegistration("/app").ConfigureAwait(false)); + Assert.AreSame(registration, await container.GetRegistration("/application").ConfigureAwait(false)); + Assert.IsNull(await container.GetRegistration("/ap").ConfigureAwait(false)); + } + + [Test] + public async Task GetRegistrationIsNullWithoutMatchingScope() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + await container.Register("/app/sw.js").ConfigureAwait(false); + + Assert.IsNull(await container.GetRegistration().ConfigureAwait(false)); + Assert.IsNull(await container.GetRegistration("/other/").ConfigureAwait(false)); + } + + [Test] + public async Task GetRegistrationsReturnsTheRegistrationsOfTheOrigin() + { + var configuration = Configuration.Default.WithNavigator().WithServiceWorkers(); + var first = GetNavigator(await OpenContextAsync(configuration, "https://first.example/").ConfigureAwait(false)).ServiceWorker(); + var second = GetNavigator(await OpenContextAsync(configuration, "https://second.example/").ConfigureAwait(false)).ServiceWorker(); + var sameOrigin = GetNavigator(await OpenContextAsync(configuration, "https://first.example/other").ConfigureAwait(false)).ServiceWorker(); + + var app = await first.Register("/app/sw.js").ConfigureAwait(false); + var root = await first.Register("/sw.js").ConfigureAwait(false); + var other = await second.Register("/sw.js").ConfigureAwait(false); + + CollectionAssert.AreEqual(new[] { root, app }, await first.GetRegistrations().ConfigureAwait(false)); + CollectionAssert.AreEqual(new[] { root, app }, await sameOrigin.GetRegistrations().ConfigureAwait(false)); + CollectionAssert.AreEqual(new[] { other }, await second.GetRegistrations().ConfigureAwait(false)); + } + + [Test] + public async Task UnregisterRemovesTheRegistration() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + var registration = await container.Register("/sw.js").ConfigureAwait(false); + var worker = registration.Active; + + Assert.IsTrue(await registration.Unregister().ConfigureAwait(false)); + Assert.IsFalse(await registration.Unregister().ConfigureAwait(false)); + Assert.AreEqual(ServiceWorkerState.Redundant, worker.State); + Assert.IsNull(registration.Active); + Assert.IsNull(await container.GetRegistration().ConfigureAwait(false)); + Assert.IsEmpty(await container.GetRegistrations().ConfigureAwait(false)); + } + + [Test] + public async Task RegisterAfterUnregisterCreatesANewRegistration() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + var first = await container.Register("/sw.js").ConfigureAwait(false); + await first.Unregister().ConfigureAwait(false); + + var second = await container.Register("/sw.js").ConfigureAwait(false); + + Assert.AreNotSame(first, second); + Assert.AreEqual(ServiceWorkerState.Activated, second.Active.State); + Assert.IsFalse(await first.Unregister().ConfigureAwait(false)); + Assert.AreSame(second, await container.GetRegistration().ConfigureAwait(false)); + } + + [Test] + public async Task ControllerIsTheActiveWorkerOfTheMatchingScope() + { + var configuration = Configuration.Default.WithNavigator().WithServiceWorkers(); + var inside = GetNavigator(await OpenContextAsync(configuration, "https://sw.example/app/page").ConfigureAwait(false)).ServiceWorker(); + var outside = GetNavigator(await OpenContextAsync(configuration, "https://sw.example/other").ConfigureAwait(false)).ServiceWorker(); + + Assert.IsNull(inside.Controller); + + var registration = await inside.Register("/app/sw.js").ConfigureAwait(false); + + Assert.AreSame(registration.Active, inside.Controller); + Assert.IsNull(outside.Controller); + } + + [Test] + public async Task RegisterRejectsCrossOriginUrls() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + + AssertSecurityError(() => container.Register("https://other.example/sw.js")); + AssertSecurityError(() => container.Register("/sw.js", new ServiceWorkerRegistrationOptions { Scope = "https://other.example/" })); + AssertSecurityError(() => container.GetRegistration("https://other.example/")); + } + + [Test] + public async Task RegisterRejectsScopesAboveTheScriptDirectory() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + + AssertSecurityError(() => container.Register("/app/sw.js", new ServiceWorkerRegistrationOptions { Scope = "/" })); + AssertSecurityError(() => container.Register("/app/sw.js", new ServiceWorkerRegistrationOptions { Scope = "/app" })); + Assert.IsEmpty(await container.GetRegistrations().ConfigureAwait(false)); + } + + [Test] + public async Task RegisterRejectsInvalidUrls() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + + Assert.ThrowsAsync(() => container.Register("ftp://sw.example/sw.js")); + Assert.ThrowsAsync(() => container.Register("/sw.js", new ServiceWorkerRegistrationOptions { Scope = "data:text/plain,x" })); + Assert.ThrowsAsync(() => container.Register("/a%2fb/sw.js")); + Assert.ThrowsAsync(() => container.Register("/sw.js", new ServiceWorkerRegistrationOptions { Scope = "/a%5Cb/" })); + } + + [TestCase("https://sw.example/", true)] + [TestCase("http://localhost:8080/", true)] + [TestCase("http://app.localhost/", true)] + [TestCase("http://127.0.0.1/", true)] + [TestCase("http://[::1]/", true)] + [TestCase("http://sw.example/", false)] + [TestCase("about:blank", false)] + public async Task ServiceWorkerContainerIsOnlyExposedInSecureContexts(String address, Boolean exposed) + { + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithServiceWorkers(), address).ConfigureAwait(false); + + Assert.AreEqual(exposed, GetNavigator(context).ServiceWorker() is not null); + } + + [Test] + public async Task ContainerRejectsInsecureDocuments() + { + var registry = new ServiceWorkerRegistry(new ServiceWorkerPlatform()); + var container = new ServiceWorkerContainer(registry, () => new Url("http://sw.example/")); + await registry.RegisterAsync(new Url("http://sw.example/sw.js"), new Url("http://sw.example/")).ConfigureAwait(false); + + AssertSecurityError(() => container.Register("http://sw.example/sw.js")); + AssertSecurityError(() => container.GetRegistration()); + AssertSecurityError(() => container.GetRegistrations()); + Assert.IsNull(container.Controller); + Assert.IsFalse(container.Ready.IsCompleted); + } + + [Test] + public async Task ReadyCompletesRightAwayWithAnActiveRegistration() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/app/").ConfigureAwait(false); + var registration = await container.Register("/app/sw.js").ConfigureAwait(false); + + var ready = container.Ready; + + Assert.IsTrue(ready.IsCompleted); + Assert.AreSame(registration, await ready.ConfigureAwait(false)); + } + + [Test] + public async Task ReadyCompletesOnceAMatchingWorkerIsActivated() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/app/page").ConfigureAwait(false); + var ready = container.Ready; + + await container.Register("/other/sw.js").ConfigureAwait(false); + Assert.IsFalse(ready.IsCompleted); + + var registration = await container.Register("/app/sw.js").ConfigureAwait(false); + var completed = await Task.WhenAny(ready, Task.Delay(TimeSpan.FromSeconds(5))).ConfigureAwait(false); + Assert.AreSame(ready, completed); + var result = await ready.ConfigureAwait(false); + + Assert.AreSame(registration, result); + Assert.AreEqual(ServiceWorkerState.Activated, result.Active.State); + } + + [Test] + public async Task ReadyStaysPendingWhenTheInstallationFails() + { + var platform = new ServiceWorkerPlatform + { + Install = _ => Task.FromException(new InvalidOperationException("broken")), + }; + + var container = await OpenContainerAsync(platform, "https://sw.example/").ConfigureAwait(false); + var ready = container.Ready; + + await container.Register("/sw.js").ConfigureAwait(false); + + Assert.IsFalse(ready.IsCompleted); + } + + [Test] + public async Task RegisterAppliesTheTypeAndUpdateViaCacheOptions() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + + var classic = await container.Register("/sw.js").ConfigureAwait(false); + Assert.AreEqual(WorkerType.Classic, classic.Active.Type); + Assert.AreEqual(ServiceWorkerUpdateViaCache.Imports, classic.UpdateViaCache); + + var module = await container.Register("/app/sw.js", new ServiceWorkerRegistrationOptions + { + Type = WorkerType.Module, + UpdateViaCache = ServiceWorkerUpdateViaCache.None, + }).ConfigureAwait(false); + Assert.AreEqual(WorkerType.Module, module.Active.Type); + Assert.AreEqual(ServiceWorkerUpdateViaCache.None, module.UpdateViaCache); + } + + [Test] + public async Task RegisteringTheSameScriptWithAnotherTypeInstallsANewWorker() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + var registration = await container.Register("/sw.js").ConfigureAwait(false); + var classic = registration.Active; + + await container.Register("/sw.js", new ServiceWorkerRegistrationOptions { Type = WorkerType.Module }).ConfigureAwait(false); + + Assert.AreEqual(ServiceWorkerState.Redundant, classic.State); + Assert.AreEqual(WorkerType.Module, registration.Active.Type); + Assert.AreEqual("https://sw.example/sw.js", registration.Active.ScriptUrl); + } + + [Test] + public async Task RegisteringTheSameScriptWithAnotherUpdateViaCacheChecksForUpdates() + { + var checks = new List(); + var platform = new ServiceWorkerPlatform + { + CheckForUpdate = worker => + { + checks.Add(worker.Registration.UpdateViaCache); + return Task.FromResult(false); + }, + }; + + var container = await OpenContainerAsync(platform, "https://sw.example/").ConfigureAwait(false); + var registration = await container.Register("/sw.js").ConfigureAwait(false); + var worker = registration.Active; + + await container.Register("/sw.js").ConfigureAwait(false); + await container.Register("/sw.js", new ServiceWorkerRegistrationOptions { UpdateViaCache = ServiceWorkerUpdateViaCache.All }).ConfigureAwait(false); + + CollectionAssert.AreEqual(new[] { ServiceWorkerUpdateViaCache.All }, checks); + Assert.AreEqual(ServiceWorkerUpdateViaCache.All, registration.UpdateViaCache); + Assert.AreSame(worker, registration.Active); + } + + [Test] + public async Task UpdateKeepsTheWorkerWhenTheScriptIsUnchanged() + { + var installs = 0; + var platform = new ServiceWorkerPlatform + { + Install = _ => + { + installs++; + return Task.CompletedTask; + }, + }; + + var container = await OpenContainerAsync(platform, "https://sw.example/").ConfigureAwait(false); + var registration = await container.Register("/sw.js").ConfigureAwait(false); + var worker = registration.Active; + + Assert.AreSame(registration, await registration.Update().ConfigureAwait(false)); + Assert.AreSame(worker, registration.Active); + Assert.AreEqual(1, installs); + } + + [Test] + public async Task UpdateInstallsANewWorkerWhenTheScriptChanged() + { + var platform = new ServiceWorkerPlatform(); + var container = await OpenContainerAsync(platform, "https://sw.example/").ConfigureAwait(false); + var registration = await container.Register("/sw.js", new ServiceWorkerRegistrationOptions { Type = WorkerType.Module }).ConfigureAwait(false); + var previous = registration.Active; + var found = 0; + var checkedWorker = default(ServiceWorker); + registration.UpdateFound += (s, e) => found++; + platform.CheckForUpdate = worker => + { + checkedWorker = worker; + return Task.FromResult(true); + }; + + await registration.Update().ConfigureAwait(false); + + Assert.AreSame(previous, checkedWorker); + Assert.AreEqual(1, found); + Assert.AreNotSame(previous, registration.Active); + Assert.AreEqual(ServiceWorkerState.Redundant, previous.State); + Assert.AreEqual(ServiceWorkerState.Activated, registration.Active.State); + Assert.AreEqual(previous.ScriptUrl, registration.Active.ScriptUrl); + Assert.AreEqual(WorkerType.Module, registration.Active.Type); + } + + [Test] + public async Task UpdateWithFailingInstallationKeepsTheActiveWorker() + { + var platform = new ServiceWorkerPlatform(); + var container = await OpenContainerAsync(platform, "https://sw.example/").ConfigureAwait(false); + var registration = await container.Register("/sw.js").ConfigureAwait(false); + var active = registration.Active; + platform.CheckForUpdate = _ => Task.FromResult(true); + platform.Install = _ => Task.FromException(new InvalidOperationException("broken")); + + await registration.Update().ConfigureAwait(false); + + Assert.AreSame(active, registration.Active); + Assert.AreEqual(ServiceWorkerState.Activated, active.State); + } + + [Test] + public async Task UpdatePropagatesFailingChecks() + { + var platform = new ServiceWorkerPlatform + { + CheckForUpdate = _ => Task.FromException(new InvalidOperationException("offline")), + }; + + var container = await OpenContainerAsync(platform, "https://sw.example/").ConfigureAwait(false); + var registration = await container.Register("/sw.js").ConfigureAwait(false); + + Assert.ThrowsAsync(() => registration.Update()); + Assert.AreEqual(ServiceWorkerState.Activated, registration.Active.State); + } + + [Test] + public async Task UpdateOfAnUnregisteredRegistrationIsRejected() + { + var container = await OpenContainerAsync(new ServiceWorkerPlatform(), "https://sw.example/").ConfigureAwait(false); + var registration = await container.Register("/sw.js").ConfigureAwait(false); + await registration.Unregister().ConfigureAwait(false); + + var ex = Assert.ThrowsAsync(() => registration.Update()); + + Assert.AreEqual((Int32)DomError.InvalidState, ex.Code); + } + + [Test] + public async Task PostMessageIsDeliveredToThePlatform() + { + var received = new List(); + var platform = new ServiceWorkerPlatform + { + Message = (worker, message) => received.Add(worker.ScriptUrl + " " + message), + }; + + var container = await OpenContainerAsync(platform, "https://sw.example/").ConfigureAwait(false); + var registration = await container.Register("/sw.js").ConfigureAwait(false); + var worker = registration.Active; + + container.Controller.PostMessage("hello"); + await registration.Unregister().ConfigureAwait(false); + worker.PostMessage("dropped"); + + CollectionAssert.AreEqual(new[] { "https://sw.example/sw.js hello" }, received); + } + + [Test] + public async Task PostMessageToClientsReachesTheControlledDocuments() + { + var registry = new ServiceWorkerRegistry(new ServiceWorkerPlatform()); + var configuration = Configuration.Default.WithNavigator().WithServiceWorkers(registry); + var inside = GetNavigator(await OpenContextAsync(configuration, "https://sw.example/app/page").ConfigureAwait(false)).ServiceWorker(); + var outside = GetNavigator(await OpenContextAsync(configuration, "https://sw.example/other").ConfigureAwait(false)).ServiceWorker(); + var insideMessages = new List(); + var outsideMessages = new List(); + inside.Message += (s, e) => insideMessages.Add((MessageEvent)e); + outside.Message += (s, e) => outsideMessages.Add((MessageEvent)e); + var registration = await inside.Register("/app/sw.js").ConfigureAwait(false); + + var count = registry.PostMessageToClients(registration.Active, "update available"); + + Assert.AreEqual(1, count); + Assert.AreEqual(1, insideMessages.Count); + Assert.AreEqual("update available", insideMessages[0].Data); + Assert.AreEqual("https://sw.example", insideMessages[0].Origin); + Assert.IsEmpty(outsideMessages); + Assert.Throws(() => registry.PostMessageToClients(null, "x")); + } + + [Test] + public async Task ControllerChangeIsRaisedForTheControlledDocuments() + { + var platform = new ServiceWorkerPlatform(); + var configuration = Configuration.Default.WithNavigator().WithServiceWorkers(platform); + var inside = GetNavigator(await OpenContextAsync(configuration, "https://sw.example/app/page").ConfigureAwait(false)).ServiceWorker(); + var outside = GetNavigator(await OpenContextAsync(configuration, "https://sw.example/other").ConfigureAwait(false)).ServiceWorker(); + var controllers = new List(); + var outsideChanges = 0; + inside.ControllerChanged += (s, e) => controllers.Add(((ServiceWorkerContainer)s).Controller); + outside.ControllerChanged += (s, e) => outsideChanges++; + + var registration = await inside.Register("/app/sw.js").ConfigureAwait(false); + var first = registration.Active; + platform.CheckForUpdate = _ => Task.FromResult(true); + await registration.Update().ConfigureAwait(false); + + CollectionAssert.AreEqual(new[] { first, registration.Active }, controllers); + Assert.AreEqual(0, outsideChanges); + } + + [Test] + public async Task FetchWithinScopeIsAnsweredByTheActiveWorker() + { + var network = new List(); + var intercepted = new List(); + var platform = new ServiceWorkerPlatform + { + Fetch = (worker, request, cancel) => + { + intercepted.Add(worker.ScriptUrl + " " + request.Address.Href); + return Task.FromResult(Respond(request.Address, "from worker")); + }, + }; + + var window = await OpenNetworkWindowAsync(platform, network, "https://sw.example/").ConfigureAwait(false); + await GetNavigator(window).ServiceWorker().Register("/app/sw.js").ConfigureAwait(false); + + var inside = await window.Fetch("/app/data.json").ConfigureAwait(false); + var outside = await window.Fetch("/data.json").ConfigureAwait(false); + + Assert.AreEqual("from worker", Read(inside)); + Assert.AreEqual("from network", Read(outside)); + CollectionAssert.AreEqual(new[] { "https://sw.example/app/sw.js https://sw.example/app/data.json" }, intercepted); + CollectionAssert.AreEqual(new[] { "https://sw.example/data.json" }, network); + } + + [Test] + public async Task FetchFallsBackToTheNetworkWhenTheWorkerDoesNotRespond() + { + var network = new List(); + var intercepted = 0; + var platform = new ServiceWorkerPlatform + { + Fetch = (worker, request, cancel) => + { + intercepted++; + return Task.FromResult(null); + }, + }; + + var window = await OpenNetworkWindowAsync(platform, network, "https://sw.example/").ConfigureAwait(false); + await GetNavigator(window).ServiceWorker().Register("/sw.js").ConfigureAwait(false); + + var response = await window.Fetch("/data.json").ConfigureAwait(false); + + Assert.AreEqual("from network", Read(response)); + Assert.AreEqual(1, intercepted); + CollectionAssert.AreEqual(new[] { "https://sw.example/data.json" }, network); + } + + [Test] + public async Task FetchIsNotInterceptedWithoutActiveRegistration() + { + var network = new List(); + var intercepted = 0; + var platform = new ServiceWorkerPlatform + { + Fetch = (worker, request, cancel) => + { + intercepted++; + return Task.FromResult(Respond(request.Address, "from worker")); + }, + }; + + var window = await OpenNetworkWindowAsync(platform, network, "https://sw.example/").ConfigureAwait(false); + + Assert.AreEqual("from network", Read(await window.Fetch("/data.json").ConfigureAwait(false))); + + var registration = await GetNavigator(window).ServiceWorker().Register("/sw.js").ConfigureAwait(false); + Assert.AreEqual("from worker", Read(await window.Fetch("/data.json").ConfigureAwait(false))); + + await registration.Unregister().ConfigureAwait(false); + Assert.AreEqual("from network", Read(await window.Fetch("/data.json").ConfigureAwait(false))); + Assert.AreEqual(1, intercepted); + Assert.AreEqual(2, network.Count); + } + + [Test] + public async Task NavigationWithinScopeIsAnsweredByTheActiveWorker() + { + var platform = new ServiceWorkerPlatform + { + Fetch = (worker, request, cancel) => Task.FromResult(Respond(request.Address, "offline")), + }; + + var configuration = Configuration.Default.WithNavigator().WithVirtualRequester(req => Respond(req.Address, "online")).WithServiceWorkers(platform); + var context = BrowsingContext.New(configuration); + var start = await context.OpenAsync("https://sw.example/").ConfigureAwait(false); + await GetNavigator(start.DefaultView).ServiceWorker().Register("/app/sw.js").ConfigureAwait(false); + + var inside = await context.OpenAsync("https://sw.example/app/page").ConfigureAwait(false); + var outside = await context.OpenAsync("https://sw.example/page").ConfigureAwait(false); + + Assert.AreEqual("online", start.Title); + Assert.AreEqual("offline", inside.Title); + Assert.AreEqual("online", outside.Title); + } + + [Test] + public void ServiceWorkersSitInFrontOfTheHttpCache() + { + var network = new Mocks.VirtualRequester(_ => null); + var cacheFirst = Configuration.Default.With(network).WithHttpCache().WithServiceWorkers(); + var workersFirst = Configuration.Default.With(network).WithServiceWorkers().WithHttpCache(); + + foreach (var configuration in new[] { cacheFirst, workersFirst }) + { + var requester = configuration.Services.OfType().Single(); + var serviceWorker = requester as ServiceWorkerRequester; + Assert.IsNotNull(serviceWorker); + Assert.IsInstanceOf(serviceWorker.Inner); + Assert.AreSame(network, ((CachingRequester)serviceWorker.Inner).Inner); + } + } + + [Test] + public void RegisteringServiceWorkersTwiceDoesNotNestTheRequesters() + { + var network = new Mocks.VirtualRequester(_ => null); + var registry = new ServiceWorkerRegistry(new ServiceWorkerPlatform()); + var configuration = Configuration.Default.With(network).WithServiceWorkers().WithServiceWorkers(registry); + + var requester = (ServiceWorkerRequester)configuration.Services.OfType().Single(); + + Assert.AreSame(network, requester.Inner); + Assert.AreSame(registry, requester.Registry); + Assert.AreSame(registry, configuration.Services.OfType().Single()); + } + + private static void AssertSecurityError(AsyncTestDelegate action) + { + var ex = Assert.ThrowsAsync(action); + Assert.AreEqual((Int32)DomError.Security, ex.Code); + } + + private static async Task OpenContainerAsync(IServiceWorkerPlatform platform, String address) + { + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithServiceWorkers(platform), address).ConfigureAwait(false); + var container = GetNavigator(context).ServiceWorker(); + Assert.IsNotNull(container); + return container; + } + + private static async Task OpenNetworkWindowAsync(IServiceWorkerPlatform platform, List network, String address) + { + var configuration = Configuration.Default + .WithNavigator() + .WithVirtualRequester(req => + { + network.Add(req.Address.Href); + return Respond(req.Address, "from network"); + }) + .WithServiceWorkers(platform); + var context = await OpenContextAsync(configuration, address).ConfigureAwait(false); + return context.Active.DefaultView; + } + + private static async Task OpenContextAsync(IConfiguration configuration, String address) + { + var context = BrowsingContext.New(configuration); + await context.OpenAsync(res => res.Address(address).Content("service-worker")).ConfigureAwait(false); + return context; + } + + private static INavigator GetNavigator(IBrowsingContext context) => GetNavigator(context?.Active?.DefaultView); + + private static INavigator GetNavigator(IWindow window) + { + var navigator = window?.Navigator; + Assert.IsNotNull(navigator); + return navigator; + } + + private static IResponse Respond(Url address, String content) => + VirtualResponse.Create(res => res.Address(address).Content(content)); + + private static String Read(IResponse response) + { + using (var reader = new StreamReader(response.Content)) + { + return reader.ReadToEnd(); + } + } + } +} diff --git a/src/AngleSharp.Io.Tests/StorageManager/StorageManagerTests.cs b/src/AngleSharp.Io.Tests/StorageManager/StorageManagerTests.cs new file mode 100644 index 0000000..6950bd1 --- /dev/null +++ b/src/AngleSharp.Io.Tests/StorageManager/StorageManagerTests.cs @@ -0,0 +1,286 @@ +namespace AngleSharp.Io.Tests.StorageManager +{ + using AngleSharp; + using AngleSharp.Browser.Dom; + using AngleSharp.Io.Dom; + using AngleSharp.Io.StorageManagerApi; + using NUnit.Framework; + using System; + using System.Threading.Tasks; + + [TestFixture] + public class StorageManagerTests + { + [Test] + public async Task StorageManagerCanBeResolvedFromNavigatorWhenRegistered() + { + var platform = new MemoryStorageManagerPlatform(); + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithStorageManager(platform), "https://storage.example/").ConfigureAwait(false); + var navigator = GetNavigator(context); + + Assert.IsNotNull(navigator.Storage()); + } + + [Test] + public async Task StorageManagerIsUnavailableWhenNotRegistered() + { + var context = await OpenContextAsync(Configuration.Default.WithNavigator(), "https://storage.example/").ConfigureAwait(false); + var navigator = GetNavigator(context); + + Assert.IsNull(navigator.Storage()); + } + + [Test] + public async Task StorageManagerIsUnavailableWhenRegisteredWithoutPlatform() + { + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithStorageManager(null), "https://storage.example/").ConfigureAwait(false); + var navigator = GetNavigator(context); + + Assert.IsNull(navigator.Storage()); + } + + [Test] + public async Task StorageManagerIsTheSameInstanceForTheSameNavigator() + { + var platform = new MemoryStorageManagerPlatform(); + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithStorageManager(platform), "https://storage.example/").ConfigureAwait(false); + var navigator = GetNavigator(context); + + Assert.AreSame(navigator.Storage(), navigator.Storage()); + } + + [Test] + public void StorageManagerProviderFactoryReturnsNullWithoutNavigator() + { + var factory = new StorageManagerProviderFactory(new MemoryStorageManagerPlatform()); + + Assert.IsNull(factory.GetStorageManager(null)); + } + + [Test] + public void StorageManagerRequiresPlatform() + { + Assert.Throws(() => new StorageManagerProviderFactory(null)); + Assert.Throws(() => new StorageManager(null, () => "https://storage.example")); + Assert.Throws(() => new StorageManager(new MemoryStorageManagerPlatform(), null)); + } + + [Test] + public async Task EstimateReturnsTheValuesOfThePlatform() + { + var platform = new MemoryStorageManagerPlatform + { + Quota = 1024 * 1024, + Usage = 4096, + }; + + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithStorageManager(platform), "https://storage.example/").ConfigureAwait(false); + var storage = GetNavigator(context).Storage(); + + var estimate = await storage.Estimate().ConfigureAwait(false); + + Assert.AreEqual(1024UL * 1024UL, estimate.Quota); + Assert.AreEqual(4096UL, estimate.Usage); + } + + [Test] + public async Task EstimateFallsBackToZeroWhenPlatformReturnsNothing() + { + var platform = new FakeStorageManagerPlatform(); + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithStorageManager(platform), "https://storage.example/").ConfigureAwait(false); + var storage = GetNavigator(context).Storage(); + + var estimate = await storage.Estimate().ConfigureAwait(false); + + Assert.IsNotNull(estimate); + Assert.AreEqual(0UL, estimate.Quota); + Assert.AreEqual(0UL, estimate.Usage); + } + + [Test] + public async Task StorageManagerPassesTheOriginOfTheDocumentToThePlatform() + { + var platform = new FakeStorageManagerPlatform + { + PersistResult = true, + PersistedResult = true, + }; + + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithStorageManager(platform), "https://storage.example:8443/some/page").ConfigureAwait(false); + var storage = GetNavigator(context).Storage(); + + await storage.Estimate().ConfigureAwait(false); + Assert.AreEqual("https://storage.example:8443", platform.LastOrigin); + + platform.LastOrigin = null; + Assert.IsTrue(await storage.Persist().ConfigureAwait(false)); + Assert.AreEqual("https://storage.example:8443", platform.LastOrigin); + + platform.LastOrigin = null; + Assert.IsTrue(await storage.Persisted().ConfigureAwait(false)); + Assert.AreEqual("https://storage.example:8443", platform.LastOrigin); + } + + [Test] + public async Task StorageManagerFollowsTheOriginAfterNavigation() + { + var platform = new FakeStorageManagerPlatform(); + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithStorageManager(platform), "https://first.example/").ConfigureAwait(false); + var storage = GetNavigator(context).Storage(); + + await context.OpenAsync(res => res.Address("https://second.example/").Content("second")).ConfigureAwait(false); + await storage.Estimate().ConfigureAwait(false); + + Assert.AreEqual("https://second.example", platform.LastOrigin); + } + + [Test] + public async Task PersistAndPersistedMapToThePlatformResults() + { + var platform = new FakeStorageManagerPlatform + { + PersistResult = false, + PersistedResult = true, + }; + + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithStorageManager(platform), "https://storage.example/").ConfigureAwait(false); + var storage = GetNavigator(context).Storage(); + + Assert.IsFalse(await storage.Persist().ConfigureAwait(false)); + Assert.IsTrue(await storage.Persisted().ConfigureAwait(false)); + Assert.AreEqual(1, platform.PersistCalls); + Assert.AreEqual(1, platform.PersistedCalls); + } + + [Test] + public async Task PersistedIsFalseBeforePersistIsRequested() + { + var platform = new MemoryStorageManagerPlatform(); + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithStorageManager(platform), "https://storage.example/").ConfigureAwait(false); + var storage = GetNavigator(context).Storage(); + + Assert.IsFalse(await storage.Persisted().ConfigureAwait(false)); + } + + [Test] + public async Task PersistIsRememberedWhenGranted() + { + var platform = new MemoryStorageManagerPlatform(); + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithStorageManager(platform), "https://storage.example/").ConfigureAwait(false); + var storage = GetNavigator(context).Storage(); + + Assert.IsTrue(await storage.Persist().ConfigureAwait(false)); + Assert.IsTrue(await storage.Persisted().ConfigureAwait(false)); + } + + [Test] + public async Task PersistIsNotRememberedWhenDenied() + { + var platform = new MemoryStorageManagerPlatform + { + AllowPersistence = false, + }; + + var context = await OpenContextAsync(Configuration.Default.WithNavigator().WithStorageManager(platform), "https://storage.example/").ConfigureAwait(false); + var storage = GetNavigator(context).Storage(); + + Assert.IsFalse(await storage.Persist().ConfigureAwait(false)); + Assert.IsFalse(await storage.Persisted().ConfigureAwait(false)); + } + + [Test] + public async Task PersistIsKeptPerOrigin() + { + var platform = new MemoryStorageManagerPlatform(); + var configuration = Configuration.Default.WithNavigator().WithStorageManager(platform); + var first = await OpenContextAsync(configuration, "https://first.example/").ConfigureAwait(false); + var second = await OpenContextAsync(configuration, "https://second.example/").ConfigureAwait(false); + var sameOrigin = await OpenContextAsync(configuration, "https://first.example/other").ConfigureAwait(false); + + await GetNavigator(first).Storage().Persist().ConfigureAwait(false); + + Assert.IsTrue(await GetNavigator(first).Storage().Persisted().ConfigureAwait(false)); + Assert.IsFalse(await GetNavigator(second).Storage().Persisted().ConfigureAwait(false)); + Assert.IsTrue(await GetNavigator(sameOrigin).Storage().Persisted().ConfigureAwait(false)); + } + + [Test] + public void StorageManagerRejectsOpaqueOrigins() + { + var platform = new FakeStorageManagerPlatform(); + var storage = new StorageManager(platform, () => String.Empty); + + Assert.ThrowsAsync(() => storage.Estimate()); + Assert.ThrowsAsync(() => storage.Persist()); + Assert.ThrowsAsync(() => storage.Persisted()); + Assert.AreEqual(0, platform.PersistCalls); + Assert.AreEqual(0, platform.PersistedCalls); + } + + [Test] + public void StorageManagerPropagatesPlatformFailures() + { + var platform = new FakeStorageManagerPlatform + { + Failure = new NotSupportedException(), + }; + + var storage = new StorageManager(platform, () => "https://storage.example"); + + Assert.ThrowsAsync(() => storage.Estimate()); + Assert.ThrowsAsync(() => storage.Persist()); + Assert.ThrowsAsync(() => storage.Persisted()); + } + + private static async Task OpenContextAsync(IConfiguration configuration, String address) + { + var context = BrowsingContext.New(configuration); + await context.OpenAsync(res => res.Address(address).Content("storage")).ConfigureAwait(false); + return context; + } + + private static INavigator GetNavigator(IBrowsingContext context) + { + var navigator = context?.Active?.DefaultView?.Navigator; + Assert.IsNotNull(navigator); + return navigator; + } + + private sealed class FakeStorageManagerPlatform : IStorageManagerPlatform + { + public StorageEstimate EstimateResult { get; set; } + + public Boolean PersistResult { get; set; } + + public Boolean PersistedResult { get; set; } + + public Exception Failure { get; set; } + + public String LastOrigin { get; set; } + + public Int32 PersistCalls { get; private set; } + + public Int32 PersistedCalls { get; private set; } + + public Task EstimateAsync(String origin) + { + LastOrigin = origin; + return Failure is null ? Task.FromResult(EstimateResult) : Task.FromException(Failure); + } + + public Task PersistAsync(String origin) + { + LastOrigin = origin; + PersistCalls++; + return Failure is null ? Task.FromResult(PersistResult) : Task.FromException(Failure); + } + + public Task PersistedAsync(String origin) + { + LastOrigin = origin; + PersistedCalls++; + return Failure is null ? Task.FromResult(PersistedResult) : Task.FromException(Failure); + } + } + } +} diff --git a/src/AngleSharp.Io.Tests/WebRtc/RtcPeerConnectionTests.cs b/src/AngleSharp.Io.Tests/WebRtc/RtcPeerConnectionTests.cs new file mode 100644 index 0000000..107fd39 --- /dev/null +++ b/src/AngleSharp.Io.Tests/WebRtc/RtcPeerConnectionTests.cs @@ -0,0 +1,972 @@ +namespace AngleSharp.Io.Tests.WebRtc +{ + using AngleSharp; + using AngleSharp.Browser; + using AngleSharp.Common; + using AngleSharp.Dom; + using AngleSharp.Dom.Events; + using AngleSharp.Io; + using AngleSharp.Io.WebRtc; + using NUnit.Framework; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + + [TestFixture] + public class RtcPeerConnectionTests + { + [Test] + public void NewConnectionStartsInTheInitialStates() + { + var connection = new RtcPeerConnection(new LoopbackRtcPlatform()); + + Assert.AreEqual(RtcSignalingState.Stable, connection.SignalingState); + Assert.AreEqual(RtcIceGatheringState.New, connection.IceGatheringState); + Assert.AreEqual(RtcIceConnectionState.New, connection.IceConnectionState); + Assert.AreEqual(RtcPeerConnectionState.New, connection.ConnectionState); + Assert.IsNull(connection.LocalDescription); + Assert.IsNull(connection.RemoteDescription); + Assert.IsNull(connection.CanTrickleIceCandidates); + Assert.Throws(() => new RtcPeerConnection((IRtcPlatform)null)); + } + + [Test] + public async Task CreateOfferReturnsAValidOfferShape() + { + var connection = new RtcPeerConnection(new LoopbackRtcPlatform()); + + var empty = await connection.CreateOffer().ConfigureAwait(false); + connection.CreateDataChannel("chat"); + var offer = await connection.CreateOffer().ConfigureAwait(false); + + Assert.AreEqual(RtcSdpType.Offer, empty.Type); + StringAssert.StartsWith("v=0\r\no=- ", empty.Sdp); + StringAssert.Contains("\r\ns=-\r\nt=0 0\r\n", empty.Sdp); + StringAssert.DoesNotContain("m=", empty.Sdp); + Assert.AreEqual(RtcSdpType.Offer, offer.Type); + StringAssert.Contains("a=group:BUNDLE 0\r\n", offer.Sdp); + StringAssert.Contains("m=application 9 UDP/DTLS/SCTP webrtc-datachannel\r\n", offer.Sdp); + StringAssert.Contains("a=ice-ufrag:", offer.Sdp); + StringAssert.Contains("a=ice-pwd:", offer.Sdp); + StringAssert.Contains("a=fingerprint:sha-256 ", offer.Sdp); + StringAssert.Contains("a=setup:actpass\r\n", offer.Sdp); + StringAssert.Contains("a=mid:0\r\n", offer.Sdp); + Assert.IsTrue(offer.Sdp.EndsWith("\r\n")); + Assert.AreEqual(RtcSignalingState.Stable, connection.SignalingState); + } + + [Test] + public async Task OfferAnswerExchangeWalksThroughTheSignalingStates() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + var callerStates = RecordSignalingStates(caller); + var calleeStates = RecordSignalingStates(callee); + caller.CreateDataChannel("chat"); + + var offer = await caller.CreateOffer().ConfigureAwait(false); + await caller.SetLocalDescription(offer).ConfigureAwait(false); + Assert.AreEqual(RtcSignalingState.HaveLocalOffer, caller.SignalingState); + Assert.AreEqual(offer.Sdp, caller.PendingLocalDescription.Sdp); + Assert.AreSame(caller.PendingLocalDescription, caller.LocalDescription); + Assert.IsNull(caller.CurrentLocalDescription); + + await callee.SetRemoteDescription(offer).ConfigureAwait(false); + Assert.AreEqual(RtcSignalingState.HaveRemoteOffer, callee.SignalingState); + Assert.AreEqual(RtcSdpType.Offer, callee.PendingRemoteDescription.Type); + + var answer = await callee.CreateAnswer().ConfigureAwait(false); + Assert.AreEqual(RtcSdpType.Answer, answer.Type); + StringAssert.Contains("a=setup:active\r\n", answer.Sdp); + StringAssert.Contains("m=application", answer.Sdp); + + await callee.SetLocalDescription(answer).ConfigureAwait(false); + await caller.SetRemoteDescription(answer).ConfigureAwait(false); + + foreach (var connection in new[] { caller, callee }) + { + Assert.AreEqual(RtcSignalingState.Stable, connection.SignalingState); + Assert.IsNull(connection.PendingLocalDescription); + Assert.IsNull(connection.PendingRemoteDescription); + Assert.IsNotNull(connection.CurrentLocalDescription); + Assert.IsNotNull(connection.CurrentRemoteDescription); + Assert.IsTrue(connection.CanTrickleIceCandidates); + } + + Assert.AreEqual(offer.Sdp, caller.CurrentLocalDescription.Sdp); + Assert.AreEqual(answer.Sdp, caller.CurrentRemoteDescription.Sdp); + CollectionAssert.AreEqual(new[] { RtcSignalingState.HaveLocalOffer, RtcSignalingState.Stable }, callerStates); + CollectionAssert.AreEqual(new[] { RtcSignalingState.HaveRemoteOffer, RtcSignalingState.Stable }, calleeStates); + } + + [Test] + public async Task MethodsRejectInvalidSignalingStates() + { + var platform = new LoopbackRtcPlatform(); + var connection = new RtcPeerConnection(platform); + var other = new RtcPeerConnection(platform); + + AssertDomError(DomError.InvalidState, () => connection.CreateAnswer()); + AssertDomError(DomError.InvalidState, () => connection.SetLocalDescription(new RtcSessionDescriptionInit { Type = RtcSdpType.Answer })); + AssertDomError(DomError.InvalidState, () => connection.SetRemoteDescription(new RtcSessionDescriptionInit { Type = RtcSdpType.Answer, Sdp = "v=0\r\n" })); + AssertDomError(DomError.InvalidState, () => connection.SetLocalDescription(new RtcSessionDescriptionInit { Type = RtcSdpType.Rollback })); + + var offer = await other.CreateOffer().ConfigureAwait(false); + await connection.SetRemoteDescription(offer).ConfigureAwait(false); + + AssertDomError(DomError.InvalidState, () => connection.CreateOffer()); + AssertDomError(DomError.InvalidState, () => connection.SetLocalDescription(new RtcSessionDescriptionInit { Type = RtcSdpType.Offer })); + Assert.AreEqual(RtcSignalingState.HaveRemoteOffer, connection.SignalingState); + } + + [Test] + public async Task SetLocalDescriptionRequiresTheLastCreatedDescription() + { + var connection = new RtcPeerConnection(new LoopbackRtcPlatform()); + var first = await connection.CreateOffer().ConfigureAwait(false); + var second = await connection.CreateOffer().ConfigureAwait(false); + + Assert.AreNotEqual(first.Sdp, second.Sdp); + AssertDomError(DomError.InvalidModification, () => connection.SetLocalDescription(first)); + AssertDomError(DomError.InvalidModification, () => connection.SetLocalDescription(new RtcSessionDescriptionInit { Type = RtcSdpType.Offer, Sdp = second.Sdp + "a=x\r\n" })); + + await connection.SetLocalDescription(new RtcSessionDescriptionInit { Type = RtcSdpType.Offer }).ConfigureAwait(false); + + Assert.AreEqual(second.Sdp, connection.LocalDescription.Sdp); + } + + [Test] + public async Task ParameterlessSetLocalDescriptionCreatesOffersAndAnswers() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + + await caller.SetLocalDescription().ConfigureAwait(false); + await callee.SetRemoteDescription(caller.LocalDescription).ConfigureAwait(false); + await callee.SetLocalDescription().ConfigureAwait(false); + await caller.SetRemoteDescription(callee.LocalDescription).ConfigureAwait(false); + + Assert.AreEqual(RtcSdpType.Offer, caller.CurrentLocalDescription.Type); + Assert.AreEqual(RtcSdpType.Answer, callee.CurrentLocalDescription.Type); + Assert.AreEqual(RtcSignalingState.Stable, caller.SignalingState); + Assert.AreEqual(RtcSignalingState.Stable, callee.SignalingState); + } + + [Test] + public async Task RollbackReturnsToStable() + { + var platform = new LoopbackRtcPlatform(); + var connection = new RtcPeerConnection(platform); + var other = new RtcPeerConnection(platform); + var states = RecordSignalingStates(connection); + + await connection.SetLocalDescription().ConfigureAwait(false); + await connection.SetLocalDescription(new RtcSessionDescriptionInit { Type = RtcSdpType.Rollback }).ConfigureAwait(false); + + Assert.AreEqual(RtcSignalingState.Stable, connection.SignalingState); + Assert.IsNull(connection.LocalDescription); + + await connection.SetRemoteDescription(await other.CreateOffer().ConfigureAwait(false)).ConfigureAwait(false); + await connection.SetRemoteDescription(new RtcSessionDescriptionInit { Type = RtcSdpType.Rollback }).ConfigureAwait(false); + + Assert.IsNull(connection.RemoteDescription); + CollectionAssert.AreEqual(new[] { RtcSignalingState.HaveLocalOffer, RtcSignalingState.Stable, RtcSignalingState.HaveRemoteOffer, RtcSignalingState.Stable }, states); + } + + [Test] + public async Task RemoteOfferDuringALocalOfferRollsBackImplicitly() + { + var platform = new LoopbackRtcPlatform(); + var polite = new RtcPeerConnection(platform); + var impolite = new RtcPeerConnection(platform); + var states = RecordSignalingStates(polite); + + await polite.SetLocalDescription().ConfigureAwait(false); + await impolite.SetLocalDescription().ConfigureAwait(false); + await polite.SetRemoteDescription(impolite.LocalDescription).ConfigureAwait(false); + + Assert.AreEqual(RtcSignalingState.HaveRemoteOffer, polite.SignalingState); + Assert.IsNull(polite.PendingLocalDescription); + CollectionAssert.AreEqual(new[] { RtcSignalingState.HaveLocalOffer, RtcSignalingState.HaveRemoteOffer }, states); + } + + [Test] + public async Task ProvisionalAnswersKeepTheNegotiationOpen() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + await caller.SetLocalDescription().ConfigureAwait(false); + await callee.SetRemoteDescription(caller.LocalDescription).ConfigureAwait(false); + var answer = await callee.CreateAnswer().ConfigureAwait(false); + + await callee.SetLocalDescription(new RtcSessionDescriptionInit { Type = RtcSdpType.PrAnswer, Sdp = answer.Sdp }).ConfigureAwait(false); + await caller.SetRemoteDescription(new RtcSessionDescriptionInit { Type = RtcSdpType.PrAnswer, Sdp = answer.Sdp }).ConfigureAwait(false); + + Assert.AreEqual(RtcSignalingState.HaveLocalPrAnswer, callee.SignalingState); + Assert.AreEqual(RtcSignalingState.HaveRemotePrAnswer, caller.SignalingState); + Assert.AreEqual(RtcSdpType.PrAnswer, callee.LocalDescription.Type); + + await callee.SetLocalDescription(answer).ConfigureAwait(false); + await caller.SetRemoteDescription(answer).ConfigureAwait(false); + + Assert.AreEqual(RtcSignalingState.Stable, callee.SignalingState); + Assert.AreEqual(RtcSignalingState.Stable, caller.SignalingState); + } + + [Test] + public void SetRemoteDescriptionValidatesTheDescription() + { + var connection = new RtcPeerConnection(new LoopbackRtcPlatform()); + + Assert.ThrowsAsync(() => connection.SetRemoteDescription((RtcSessionDescriptionInit)null)); + Assert.ThrowsAsync(() => connection.SetRemoteDescription(new RtcSessionDescriptionInit { Sdp = "v=0\r\n" })); + AssertDomError(DomError.Syntax, () => connection.SetRemoteDescription(new RtcSessionDescriptionInit { Type = RtcSdpType.Offer })); + AssertDomError(DomError.Syntax, () => connection.SetRemoteDescription(new RtcSessionDescriptionInit { Type = RtcSdpType.Offer, Sdp = "garbage" })); + Assert.AreEqual(RtcSignalingState.Stable, connection.SignalingState); + } + + [Test] + public async Task CloseEndsTheConnectionWithoutEvents() + { + var connection = new RtcPeerConnection(new LoopbackRtcPlatform()); + var channel = connection.CreateDataChannel("chat"); + var events = RecordAllEvents(connection); + await connection.SetLocalDescription().ConfigureAwait(false); + events.Clear(); + + connection.Close(); + connection.Close(); + + Assert.AreEqual(RtcSignalingState.Closed, connection.SignalingState); + Assert.AreEqual(RtcIceConnectionState.Closed, connection.IceConnectionState); + Assert.AreEqual(RtcPeerConnectionState.Closed, connection.ConnectionState); + Assert.AreEqual(RtcDataChannelState.Closed, channel.ReadyState); + Assert.IsEmpty(events); + AssertDomError(DomError.InvalidState, () => connection.CreateOffer()); + AssertDomError(DomError.InvalidState, () => connection.SetLocalDescription()); + AssertDomError(DomError.InvalidState, () => connection.AddIceCandidate()); + Assert.Throws(() => connection.CreateDataChannel("other")); + Assert.Throws(() => connection.SetConfiguration(new RtcConfiguration())); + } + + [Test] + public async Task SettingTheLocalDescriptionGathersCandidates() + { + var connection = new RtcPeerConnection(new LoopbackRtcPlatform()); + var events = RecordAllEvents(connection); + var candidates = new List(); + connection.IceCandidate += (s, e) => candidates.Add(((RtcPeerConnectionIceEvent)e).Candidate); + connection.CreateDataChannel("chat"); + events.Clear(); + + await connection.SetLocalDescription().ConfigureAwait(false); + + CollectionAssert.AreEqual(new[] + { + "signalingstatechange", + "icegatheringstatechange:gathering", + "icecandidate", + "icegatheringstatechange:complete", + "icecandidate:end", + }, events); + Assert.AreEqual(RtcIceGatheringState.Complete, connection.IceGatheringState); + + var candidate = candidates[0]; + var ufrag = connection.LocalDescription.Sdp.Split('\n').First(m => m.StartsWith("a=ice-ufrag:")).Substring(12).TrimEnd('\r'); + StringAssert.StartsWith("candidate:1 1 udp 2122260223 127.0.0.1 ", candidate.Candidate); + Assert.AreEqual("0", candidate.SdpMid); + Assert.AreEqual((UInt16)0, candidate.SdpMLineIndex); + Assert.AreEqual("1", candidate.Foundation); + Assert.AreEqual(RtcIceComponent.Rtp, candidate.Component); + Assert.AreEqual(RtcIceProtocol.Udp, candidate.Protocol); + Assert.AreEqual(2122260223u, candidate.Priority); + Assert.AreEqual("127.0.0.1", candidate.Address); + Assert.IsNotNull(candidate.Port); + Assert.AreEqual(RtcIceCandidateType.Host, candidate.Type); + Assert.AreEqual(ufrag, candidate.UsernameFragment); + Assert.IsNull(candidates[1]); + } + + [Test] + public async Task CandidateEventsSeeTheAppliedDescription() + { + var connection = new RtcPeerConnection(new LoopbackRtcPlatform()); + var observed = default(RtcSignalingState?); + var description = default(RtcSessionDescription); + connection.IceCandidate += (s, e) => + { + observed ??= connection.SignalingState; + description ??= connection.LocalDescription; + }; + connection.CreateDataChannel("chat"); + + await connection.SetLocalDescription().ConfigureAwait(false); + + Assert.AreEqual(RtcSignalingState.HaveLocalOffer, observed); + Assert.IsNotNull(description); + } + + [Test] + public async Task ConnectionsWithoutMediaDescriptionsDoNotGather() + { + var connection = new RtcPeerConnection(new LoopbackRtcPlatform()); + var candidates = 0; + connection.IceCandidate += (s, e) => candidates++; + + await connection.SetLocalDescription().ConfigureAwait(false); + + Assert.AreEqual(RtcIceGatheringState.New, connection.IceGatheringState); + Assert.AreEqual(0, candidates); + } + + [Test] + public async Task ExchangingCandidatesConnectsThePeers() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + var callerEvents = RecordAllEvents(caller); + var calleeEvents = RecordAllEvents(callee); + caller.CreateDataChannel("chat"); + + await ConnectAsync(caller, callee).ConfigureAwait(false); + + foreach (var connection in new[] { caller, callee }) + { + Assert.AreEqual(RtcIceConnectionState.Connected, connection.IceConnectionState); + Assert.AreEqual(RtcPeerConnectionState.Connected, connection.ConnectionState); + } + + var expected = new[] + { + "iceconnectionstatechange:checking", + "connectionstatechange:connecting", + "iceconnectionstatechange:connected", + "connectionstatechange:connected", + }; + CollectionAssert.AreEqual(expected, callerEvents.Where(m => m.Contains("connectionstatechange"))); + CollectionAssert.AreEqual(expected, calleeEvents.Where(m => m.Contains("connectionstatechange"))); + } + + [Test] + public async Task CandidatesOfUnknownPeersDoNotConnect() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + caller.CreateDataChannel("chat"); + await caller.SetLocalDescription().ConfigureAwait(false); + await callee.SetRemoteDescription(caller.LocalDescription).ConfigureAwait(false); + await callee.SetLocalDescription().ConfigureAwait(false); + await caller.SetRemoteDescription(callee.LocalDescription).ConfigureAwait(false); + + await caller.AddIceCandidate(new RtcIceCandidateInit { Candidate = "candidate:1 1 udp 2122260223 192.0.2.1 1234 typ host", SdpMid = "0" }).ConfigureAwait(false); + await caller.AddIceCandidate().ConfigureAwait(false); + + Assert.AreEqual(RtcIceConnectionState.New, caller.IceConnectionState); + } + + [Test] + public async Task AddIceCandidateValidatesItsInput() + { + var platform = new LoopbackRtcPlatform(); + var connection = new RtcPeerConnection(platform); + var candidate = new RtcIceCandidateInit { Candidate = "candidate:1 1 udp 1 127.0.0.1 9 typ host", SdpMid = "0" }; + + AssertDomError(DomError.InvalidState, () => connection.AddIceCandidate(candidate)); + Assert.ThrowsAsync(() => connection.AddIceCandidate(new RtcIceCandidateInit { Candidate = candidate.Candidate })); + + await connection.SetRemoteDescription(await new RtcPeerConnection(platform).CreateOffer().ConfigureAwait(false)).ConfigureAwait(false); + + await connection.AddIceCandidate(candidate).ConfigureAwait(false); + await connection.AddIceCandidate(new RtcIceCandidateInit()).ConfigureAwait(false); + await connection.AddIceCandidate().ConfigureAwait(false); + } + + [Test] + public async Task RelayPolicyGathersNoHostCandidates() + { + var platform = new LoopbackRtcPlatform(); + var configuration = new RtcConfiguration { IceTransportPolicy = RtcIceTransportPolicy.Relay }; + var caller = new RtcPeerConnection(platform, configuration); + var callee = new RtcPeerConnection(platform, configuration); + var candidates = new List(); + caller.IceCandidate += (s, e) => candidates.Add(((RtcPeerConnectionIceEvent)e).Candidate); + caller.CreateDataChannel("chat"); + + await ConnectAsync(caller, callee).ConfigureAwait(false); + + CollectionAssert.AreEqual(new RtcIceCandidate[] { null }, candidates); + Assert.AreEqual(RtcIceGatheringState.Complete, caller.IceGatheringState); + Assert.AreEqual(RtcIceConnectionState.New, caller.IceConnectionState); + } + + [Test] + public void IceCandidatesAreParsed() + { + var srflx = new RtcIceCandidate(new RtcIceCandidateInit + { + Candidate = "candidate:842163049 1 UDP 1677729535 203.0.113.7 61665 typ srflx raddr 192.168.1.2 rport 61665 generation 0 ufrag abcd", + SdpMLineIndex = 0, + }); + var tcp = new RtcIceCandidate(new RtcIceCandidateInit + { + Candidate = "candidate:3 2 tcp 1518280447 192.168.1.2 9 typ host tcptype active", + SdpMid = "1", + UsernameFragment = "given", + }); + var broken = new RtcIceCandidate(new RtcIceCandidateInit { Candidate = "candidate:x y", SdpMid = "0" }); + + Assert.AreEqual("842163049", srflx.Foundation); + Assert.AreEqual(RtcIceProtocol.Udp, srflx.Protocol); + Assert.AreEqual(RtcIceCandidateType.Srflx, srflx.Type); + Assert.AreEqual("192.168.1.2", srflx.RelatedAddress); + Assert.AreEqual((UInt16)61665, srflx.RelatedPort); + Assert.AreEqual("abcd", srflx.UsernameFragment); + Assert.IsNull(srflx.SdpMid); + Assert.AreEqual(RtcIceComponent.Rtcp, tcp.Component); + Assert.AreEqual(RtcIceProtocol.Tcp, tcp.Protocol); + Assert.AreEqual(RtcIceTcpCandidateType.Active, tcp.TcpType); + Assert.AreEqual("given", tcp.UsernameFragment); + Assert.IsNull(broken.Foundation); + Assert.IsNull(broken.Port); + Assert.IsNull(broken.Type); + Assert.Throws(() => new RtcIceCandidate(new RtcIceCandidateInit { Candidate = "candidate:1" })); + Assert.Throws(() => new RtcIceCandidate(null)); + + var json = tcp.ToJson(); + Assert.AreEqual(tcp.Candidate, json.Candidate); + Assert.AreEqual("1", json.SdpMid); + Assert.IsNull(json.SdpMLineIndex); + Assert.AreEqual("given", json.UsernameFragment); + } + + [Test] + public void SessionDescriptionsRoundTrip() + { + var description = new RtcSessionDescription(new RtcSessionDescriptionInit { Type = RtcSdpType.Answer, Sdp = "v=0\r\n" }); + var json = description.ToJson(); + + Assert.AreEqual(RtcSdpType.Answer, description.Type); + Assert.AreEqual("v=0\r\n", description.Sdp); + Assert.AreEqual(RtcSdpType.Answer, json.Type); + Assert.AreEqual("v=0\r\n", json.Sdp); + Assert.Throws(() => new RtcSessionDescription(new RtcSessionDescriptionInit())); + } + + [Test] + public async Task CreatingADataChannelNeedsNegotiationOnce() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + var needed = 0; + caller.NegotiationNeeded += (s, e) => needed++; + + caller.CreateDataChannel("one"); + caller.CreateDataChannel("two"); + Assert.AreEqual(1, needed); + + await ConnectAsync(caller, callee).ConfigureAwait(false); + caller.CreateDataChannel("three"); + + Assert.AreEqual(1, needed); + } + + [Test] + public async Task NegotiationNeededWaitsForTheStableState() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + var needed = 0; + callee.NegotiationNeeded += (s, e) => needed++; + await caller.SetLocalDescription().ConfigureAwait(false); + await callee.SetRemoteDescription(caller.LocalDescription).ConfigureAwait(false); + + callee.CreateDataChannel("late"); + Assert.AreEqual(0, needed); + + await callee.SetLocalDescription().ConfigureAwait(false); + + Assert.AreEqual(1, needed); + } + + [Test] + public async Task RestartIceUsesNewCredentials() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + caller.CreateDataChannel("chat"); + await ConnectAsync(caller, callee).ConfigureAwait(false); + var needed = 0; + var gathered = new List(); + caller.NegotiationNeeded += (s, e) => needed++; + caller.IceCandidate += (s, e) => gathered.Add(((RtcPeerConnectionIceEvent)e).Candidate?.UsernameFragment); + var before = GetUfrag(caller.LocalDescription); + var calleeBefore = GetUfrag(callee.LocalDescription); + + caller.RestartIce(); + Assert.AreEqual(1, needed); + + var offer = await caller.CreateOffer().ConfigureAwait(false); + await caller.SetLocalDescription(offer).ConfigureAwait(false); + await callee.SetRemoteDescription(offer).ConfigureAwait(false); + var answer = await callee.CreateAnswer().ConfigureAwait(false); + await callee.SetLocalDescription(answer).ConfigureAwait(false); + await caller.SetRemoteDescription(answer).ConfigureAwait(false); + + Assert.AreNotEqual(before, GetUfrag(caller.LocalDescription)); + Assert.AreNotEqual(calleeBefore, GetUfrag(answer.Sdp)); + CollectionAssert.AreEqual(new[] { GetUfrag(caller.LocalDescription), null }, gathered); + Assert.AreEqual(1, needed); + Assert.AreEqual(RtcPeerConnectionState.Connected, caller.ConnectionState); + + var again = await caller.CreateOffer().ConfigureAwait(false); + Assert.AreEqual(GetUfrag(caller.LocalDescription), GetUfrag(again.Sdp)); + Assert.AreEqual(GetUfrag(callee.LocalDescription), GetUfrag(caller.RemoteDescription)); + } + + [Test] + public async Task AnnouncedDataChannelsOpenOnBothPeers() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + var local = caller.CreateDataChannel("chat", new RtcDataChannelInit { Ordered = false, MaxRetransmits = 3, Protocol = "json" }); + var opened = 0; + var remote = default(RtcDataChannel); + var remoteState = default(RtcDataChannelState?); + local.Opened += (s, e) => opened++; + callee.DataChannel += (s, e) => + { + remote = ((RtcDataChannelEvent)e).Channel; + remoteState = remote.ReadyState; + }; + + Assert.AreEqual(RtcDataChannelState.Connecting, local.ReadyState); + Assert.IsNull(local.Id); + + await ConnectAsync(caller, callee).ConfigureAwait(false); + + Assert.AreEqual(1, opened); + Assert.AreEqual(RtcDataChannelState.Open, local.ReadyState); + Assert.IsNotNull(remote); + Assert.AreEqual(RtcDataChannelState.Open, remoteState); + Assert.AreEqual("chat", remote.Label); + Assert.AreEqual("json", remote.Protocol); + Assert.IsFalse(remote.Ordered); + Assert.AreEqual((UInt16)3, remote.MaxRetransmits); + Assert.IsFalse(remote.Negotiated); + Assert.AreEqual(local.Id, remote.Id); + Assert.AreEqual((UInt16)1, local.Id); + } + + [Test] + public async Task DataChannelsExchangeMessages() + { + var (local, remote) = await OpenChannelPairAsync().ConfigureAwait(false); + var atRemote = RecordMessages(remote); + var atLocal = RecordMessages(local); + var bytes = new Byte[] { 1, 2, 3 }; + + local.Send("hello"); + local.Send(bytes); + bytes[0] = 9; + remote.Send("back"); + + Assert.AreEqual(2, atRemote.Count); + Assert.AreEqual("hello", atRemote[0]); + CollectionAssert.AreEqual(new Byte[] { 1, 2, 3 }, (Byte[])atRemote[1]); + CollectionAssert.AreEqual(new Object[] { "back" }, atLocal); + Assert.AreEqual(0u, local.BufferedAmount); + } + + [Test] + public async Task MessagesSentWhenOpeningAreNotLost() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + var local = caller.CreateDataChannel("chat"); + var received = new List(); + local.Opened += (s, e) => local.Send("first"); + callee.DataChannel += (s, e) => + { + var channel = ((RtcDataChannelEvent)e).Channel; + channel.Message += (s2, e2) => received.Add(((MessageEvent)e2).Data); + }; + + await ConnectAsync(caller, callee).ConfigureAwait(false); + + CollectionAssert.AreEqual(new Object[] { "first" }, received); + } + + [Test] + public async Task NegotiatedDataChannelsOpenWithoutAnnouncement() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + var local = caller.CreateDataChannel("pre", new RtcDataChannelInit { Negotiated = true, Id = 5 }); + var remote = callee.CreateDataChannel("pre", new RtcDataChannelInit { Negotiated = true, Id = 5 }); + var announced = 0; + callee.DataChannel += (s, e) => announced++; + + await ConnectAsync(caller, callee).ConfigureAwait(false); + var received = RecordMessages(remote); + local.Send("ping"); + + Assert.AreEqual(RtcDataChannelState.Open, local.ReadyState); + Assert.AreEqual(RtcDataChannelState.Open, remote.ReadyState); + Assert.AreEqual((UInt16)5, remote.Id); + Assert.AreEqual(0, announced); + CollectionAssert.AreEqual(new Object[] { "ping" }, received); + } + + [Test] + public async Task ChannelsCreatedOnAnEstablishedConnectionOpenRightAway() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + caller.CreateDataChannel("first"); + await ConnectAsync(caller, callee).ConfigureAwait(false); + var announced = new List(); + caller.DataChannel += (s, e) => announced.Add(((RtcDataChannelEvent)e).Channel); + + var second = callee.CreateDataChannel("second"); + + Assert.AreEqual(RtcDataChannelState.Open, second.ReadyState); + Assert.AreEqual((UInt16)0, second.Id); + Assert.AreEqual("second", announced.Single().Label); + } + + [Test] + public async Task ClosingAChannelClosesItOnBothPeers() + { + var (local, remote) = await OpenChannelPairAsync().ConfigureAwait(false); + var localEvents = RecordChannelEvents(local); + var remoteEvents = RecordChannelEvents(remote); + + local.Close(); + local.Close(); + + Assert.AreEqual(RtcDataChannelState.Closed, local.ReadyState); + Assert.AreEqual(RtcDataChannelState.Closed, remote.ReadyState); + CollectionAssert.AreEqual(new[] { "close" }, localEvents); + CollectionAssert.AreEqual(new[] { "closing", "close" }, remoteEvents); + Assert.Throws(() => local.Send("late")); + } + + [Test] + public void SendingBeforeTheChannelIsOpenFails() + { + var channel = new RtcPeerConnection(new LoopbackRtcPlatform()).CreateDataChannel("chat"); + + var ex = Assert.Throws(() => channel.Send("too early")); + + Assert.AreEqual((Int32)DomError.InvalidState, ex.Code); + } + + [Test] + public async Task ClosingAConnectionDisconnectsTheRemotePeer() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + var local = caller.CreateDataChannel("chat"); + var remote = default(RtcDataChannel); + callee.DataChannel += (s, e) => remote = ((RtcDataChannelEvent)e).Channel; + await ConnectAsync(caller, callee).ConfigureAwait(false); + var remoteEvents = RecordChannelEvents(remote); + + caller.Close(); + + Assert.AreEqual(RtcDataChannelState.Closed, local.ReadyState); + Assert.AreEqual(RtcIceConnectionState.Disconnected, callee.IceConnectionState); + Assert.AreEqual(RtcPeerConnectionState.Disconnected, callee.ConnectionState); + Assert.AreEqual(RtcDataChannelState.Closed, remote.ReadyState); + CollectionAssert.AreEqual(new[] { "closing", "close" }, remoteEvents); + } + + [Test] + public void CreateDataChannelValidatesItsOptions() + { + var connection = new RtcPeerConnection(new LoopbackRtcPlatform()); + + Assert.Throws(() => connection.CreateDataChannel("x", new RtcDataChannelInit { Negotiated = true })); + Assert.Throws(() => connection.CreateDataChannel("x", new RtcDataChannelInit { MaxPacketLifeTime = 1, MaxRetransmits = 1 })); + Assert.Throws(() => connection.CreateDataChannel("x", new RtcDataChannelInit { Negotiated = true, Id = 65535 })); + Assert.Throws(() => connection.CreateDataChannel(new String('x', 65536))); + + var channel = connection.CreateDataChannel(null, new RtcDataChannelInit { Id = 7 }); + Assert.AreEqual(String.Empty, channel.Label); + Assert.IsNull(channel.Id); + Assert.AreEqual("arraybuffer", channel.BinaryType); + channel.BinaryType = "blob"; + channel.BinaryType = "invalid"; + Assert.AreEqual("blob", channel.BinaryType); + } + + [Test] + public void ConfigurationIsValidated() + { + var platform = new LoopbackRtcPlatform(); + + AssertDomError(DomError.Syntax, () => new RtcPeerConnection(platform, Servers(new RtcIceServer { Urls = new[] { "http://stun.example" } }))); + AssertDomError(DomError.Syntax, () => new RtcPeerConnection(platform, Servers(new RtcIceServer()))); + AssertDomError(DomError.Syntax, () => new RtcPeerConnection(platform, Servers(new RtcIceServer { Urls = new[] { "stun:" } }))); + AssertDomError(DomError.InvalidAccess, () => new RtcPeerConnection(platform, Servers(new RtcIceServer { Urls = new[] { "turn:turn.example" } }))); + + var configuration = Servers( + new RtcIceServer { Urls = new[] { "stun:stun.example:3478", "STUNS:stun.example" } }, + new RtcIceServer { Urls = new[] { "turn:turn.example?transport=tcp" }, Username = "user", Credential = "secret" }); + var connection = new RtcPeerConnection(platform, configuration); + configuration.IceServers[0].Urls[0] = "changed"; + + var copy = connection.GetConfiguration(); + Assert.AreEqual("stun:stun.example:3478", copy.IceServers[0].Urls[0]); + Assert.AreEqual("user", copy.IceServers[1].Username); + copy.IceServers[0].Urls[0] = "changed"; + Assert.AreEqual("stun:stun.example:3478", connection.GetConfiguration().IceServers[0].Urls[0]); + } + + [Test] + public async Task SetConfigurationKeepsTheFixedPolicies() + { + var connection = new RtcPeerConnection(new LoopbackRtcPlatform(), new RtcConfiguration { IceCandidatePoolSize = 1 }); + + AssertDomError(DomError.InvalidModification, () => connection.SetConfiguration(new RtcConfiguration { BundlePolicy = RtcBundlePolicy.MaxBundle, IceCandidatePoolSize = 1 })); + connection.SetConfiguration(new RtcConfiguration { IceTransportPolicy = RtcIceTransportPolicy.Relay, IceCandidatePoolSize = 2 }); + Assert.AreEqual(RtcIceTransportPolicy.Relay, connection.GetConfiguration().IceTransportPolicy); + + await connection.SetLocalDescription().ConfigureAwait(false); + + AssertDomError(DomError.InvalidModification, () => connection.SetConfiguration(new RtcConfiguration { IceCandidatePoolSize = 3 })); + AssertDomError(DomError.Syntax, () => connection.SetConfiguration(new RtcConfiguration { IceCandidatePoolSize = 2, IceServers = new[] { new RtcIceServer { Urls = new[] { "x" } } } })); + Assert.AreEqual(2, connection.GetConfiguration().IceCandidatePoolSize); + } + + [Test] + public async Task TransportEventsAreQueuedOnTheEventLoop() + { + var loop = new ManualEventLoop(); + var connection = new RtcPeerConnection(new LoopbackRtcPlatform(), null, loop); + var events = RecordAllEvents(connection); + connection.CreateDataChannel("chat"); + + Assert.IsEmpty(events); + loop.RunAll(); + CollectionAssert.AreEqual(new[] { "negotiationneeded" }, events); + events.Clear(); + + await connection.SetLocalDescription().ConfigureAwait(false); + + CollectionAssert.AreEqual(new[] { "signalingstatechange" }, events); + Assert.AreEqual(RtcIceGatheringState.New, connection.IceGatheringState); + + loop.RunAll(); + + Assert.AreEqual(RtcIceGatheringState.Complete, connection.IceGatheringState); + CollectionAssert.AreEqual(new[] + { + "signalingstatechange", + "icegatheringstatechange:gathering", + "icecandidate", + "icegatheringstatechange:complete", + "icecandidate:end", + }, events); + } + + [Test] + public async Task PeersConnectThroughTheirEventLoops() + { + var platform = new LoopbackRtcPlatform(); + var loop = new ManualEventLoop(); + var caller = new RtcPeerConnection(platform, null, loop); + var callee = new RtcPeerConnection(platform, null, loop); + var local = caller.CreateDataChannel("chat"); + var received = new List(); + callee.DataChannel += (s, e) => ((RtcDataChannelEvent)e).Channel.Message += (s2, e2) => received.Add(((MessageEvent)e2).Data); + local.Opened += (s, e) => local.Send("over the loop"); + + await ConnectAsync(caller, callee, loop.RunAll).ConfigureAwait(false); + loop.RunAll(); + + Assert.AreEqual(RtcPeerConnectionState.Connected, caller.ConnectionState); + Assert.AreEqual(RtcPeerConnectionState.Connected, callee.ConnectionState); + CollectionAssert.AreEqual(new Object[] { "over the loop" }, received); + } + + [Test] + public async Task WindowsUseTheRegisteredPlatformAndEventLoop() + { + var loop = new ManualEventLoop(); + var configuration = Configuration.Default.WithWebRtc().WithOnly(_ => loop); + var context = BrowsingContext.New(configuration); + var document = await context.OpenAsync(res => res.Address("https://rtc.example/").Content("")).ConfigureAwait(false); + loop.RunAll(); + var caller = new RtcPeerConnection(document.DefaultView); + var callee = new RtcPeerConnection(document.DefaultView, new RtcConfiguration()); + caller.CreateDataChannel("chat"); + + await ConnectAsync(caller, callee, loop.RunAll).ConfigureAwait(false); + + Assert.AreEqual(RtcPeerConnectionState.Connected, caller.ConnectionState); + Assert.AreEqual(RtcPeerConnectionState.Connected, callee.ConnectionState); + } + + [Test] + public async Task WindowsWithoutPlatformCannotCreateConnections() + { + var context = BrowsingContext.New(Configuration.Default.WithWebRtc(null)); + var document = await context.OpenAsync(res => res.Content("")).ConfigureAwait(false); + + var ex = Assert.Throws(() => new RtcPeerConnection(document.DefaultView)); + + Assert.AreEqual((Int32)DomError.NotSupported, ex.Code); + Assert.Throws(() => new RtcPeerConnection((IWindow)null)); + } + + [Test] + public void WithIoRegistersTheWebRtcPlatform() + { + var platform = new LoopbackRtcPlatform(); + + var configuration = Configuration.Default.WithIo(new IoOptions { WebRtcPlatform = platform }); + + Assert.AreSame(platform, configuration.Services.OfType().Single()); + Assert.IsEmpty(Configuration.Default.WithIo(new IoOptions()).Services.OfType()); + } + + private static async Task<(RtcDataChannel Local, RtcDataChannel Remote)> OpenChannelPairAsync() + { + var platform = new LoopbackRtcPlatform(); + var caller = new RtcPeerConnection(platform); + var callee = new RtcPeerConnection(platform); + var local = caller.CreateDataChannel("chat"); + var remote = default(RtcDataChannel); + callee.DataChannel += (s, e) => remote = ((RtcDataChannelEvent)e).Channel; + await ConnectAsync(caller, callee).ConfigureAwait(false); + Assert.IsNotNull(remote); + return (local, remote); + } + + private static async Task ConnectAsync(RtcPeerConnection caller, RtcPeerConnection callee, Action spin = null) + { + // candidates are buffered until the remote description is set, + // like signaling code has to do + var fromCaller = new List(); + var fromCallee = new List(); + caller.IceCandidate += (s, e) => fromCaller.Add(((RtcPeerConnectionIceEvent)e).Candidate); + callee.IceCandidate += (s, e) => fromCallee.Add(((RtcPeerConnectionIceEvent)e).Candidate); + spin ??= () => { }; + + var offer = await caller.CreateOffer().ConfigureAwait(false); + await caller.SetLocalDescription(offer).ConfigureAwait(false); + await callee.SetRemoteDescription(offer).ConfigureAwait(false); + var answer = await callee.CreateAnswer().ConfigureAwait(false); + await callee.SetLocalDescription(answer).ConfigureAwait(false); + await caller.SetRemoteDescription(answer).ConfigureAwait(false); + spin.Invoke(); + + foreach (var candidate in fromCaller.ToArray()) + { + await callee.AddIceCandidate(candidate).ConfigureAwait(false); + } + + foreach (var candidate in fromCallee.ToArray()) + { + await caller.AddIceCandidate(candidate).ConfigureAwait(false); + } + + spin.Invoke(); + } + + private static String GetUfrag(RtcSessionDescription description) => GetUfrag(description?.Sdp); + + private static String GetUfrag(String sdp) => + sdp?.Split('\n').Where(m => m.StartsWith("a=ice-ufrag:")).Select(m => m.Substring(12).TrimEnd('\r')).FirstOrDefault(); + + private static RtcConfiguration Servers(params RtcIceServer[] servers) => new RtcConfiguration { IceServers = servers }; + + private static List RecordSignalingStates(RtcPeerConnection connection) + { + var states = new List(); + connection.SignalingStateChanged += (s, e) => states.Add(connection.SignalingState); + return states; + } + + private static List RecordAllEvents(RtcPeerConnection connection) + { + var events = new List(); + connection.NegotiationNeeded += (s, e) => events.Add(e.Type); + connection.SignalingStateChanged += (s, e) => events.Add(e.Type); + connection.IceGatheringStateChanged += (s, e) => events.Add(e.Type + ":" + Literal(connection.IceGatheringState)); + connection.IceConnectionStateChanged += (s, e) => events.Add(e.Type + ":" + Literal(connection.IceConnectionState)); + connection.ConnectionStateChanged += (s, e) => events.Add(e.Type + ":" + Literal(connection.ConnectionState)); + connection.IceCandidate += (s, e) => events.Add(((RtcPeerConnectionIceEvent)e).Candidate is null ? e.Type + ":end" : e.Type); + connection.DataChannel += (s, e) => events.Add(e.Type); + return events; + } + + private static List RecordMessages(RtcDataChannel channel) + { + var messages = new List(); + channel.Message += (s, e) => messages.Add(((MessageEvent)e).Data); + return messages; + } + + private static List RecordChannelEvents(RtcDataChannel channel) + { + var events = new List(); + channel.Closing += (s, e) => events.Add(e.Type); + channel.Closed += (s, e) => events.Add(e.Type); + return events; + } + + private static String Literal(T value) => value.ToString().ToLowerInvariant(); + + private static void AssertDomError(DomError error, AsyncTestDelegate action) + { + var ex = Assert.ThrowsAsync(action); + Assert.AreEqual((Int32)error, ex.Code); + } + + private static void AssertDomError(DomError error, TestDelegate action) + { + var ex = Assert.Throws(action); + Assert.AreEqual((Int32)error, ex.Code); + } + + private sealed class ManualEventLoop : IEventLoop + { + private readonly Queue> _tasks = new Queue>(); + + public ICancellable Enqueue(Action action, TaskPriority priority) + { + _tasks.Enqueue(action); + return null; + } + + public void RunAll() + { + while (_tasks.Count > 0) + { + _tasks.Dequeue().Invoke(CancellationToken.None); + } + } + + public void CancelAll() => _tasks.Clear(); + + public void Spin() => RunAll(); + } + } +} diff --git a/src/AngleSharp.Io/BeaconOptions.cs b/src/AngleSharp.Io/BeaconOptions.cs new file mode 100644 index 0000000..bb1c9cd --- /dev/null +++ b/src/AngleSharp.Io/BeaconOptions.cs @@ -0,0 +1,29 @@ +namespace AngleSharp.Io; + +using System; + +/// +/// Options for sending beacons via navigator.sendBeacon. +/// +public sealed class BeaconOptions +{ + /// + /// The quota used by browsers, i.e., 64 KiB. + /// + public const Int64 DefaultQuota = 64 * 1024; + + /// + /// Gets or sets the maximum number of body bytes that may be pending, + /// i.e., queued or in flight, per browsing context. A beacon exceeding + /// the quota is rejected. + /// + public Int64 Quota { get; set; } = DefaultQuota; + + /// + /// Gets or sets the maximum number of beacons sent at the same time per + /// browsing context. Further beacons are queued and sent in order once + /// a running one completes. By default, this is 0, which means that all + /// beacons are sent right away. + /// + public Int32 MaxConcurrentRequests { get; set; } +} diff --git a/src/AngleSharp.Io/Clipboard/ClipboardProviderFactory.cs b/src/AngleSharp.Io/Clipboard/ClipboardProviderFactory.cs index 30c8b4f..b915705 100644 --- a/src/AngleSharp.Io/Clipboard/ClipboardProviderFactory.cs +++ b/src/AngleSharp.Io/Clipboard/ClipboardProviderFactory.cs @@ -31,7 +31,7 @@ public Clipboard GetClipboard(INavigator navigator) return null; } - return _clipboards.GetValue(navigator, _ => new Clipboard(_platform)); + return _clipboards.GetValue(navigator, _ => new Clipboard(_platform, navigator.Permissions())); } } } \ No newline at end of file diff --git a/src/AngleSharp.Io/Dom/BeaconSender.cs b/src/AngleSharp.Io/Dom/BeaconSender.cs new file mode 100644 index 0000000..087ff11 --- /dev/null +++ b/src/AngleSharp.Io/Dom/BeaconSender.cs @@ -0,0 +1,315 @@ +namespace AngleSharp.Io.Dom +{ + using AngleSharp.Dom; + using AngleSharp.Io; + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Runtime.CompilerServices; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Sends beacons and keeps track of the pending bytes per browsing + /// context, which stands in for the spec's fetch group. + /// + sealed class BeaconSender + { + private static readonly ConditionalWeakTable Senders = new ConditionalWeakTable(); + + private static readonly String[] SafelistedContentTypes = new[] + { + "application/x-www-form-urlencoded", + "multipart/form-data", + "text/plain", + }; + + private readonly IBrowsingContext _context; + private readonly Queue _queue; + + private Int64 _pending; + private Int32 _running; + + private BeaconSender(IBrowsingContext context) + { + _context = context; + _queue = new Queue(); + } + + public static Boolean Send(IBrowsingContext context, String url, Object data) + { + var document = context?.Active; + + if (document is null) + { + return false; + } + + var documentUrl = new Url(document.Url); + var target = new Url(document.BaseUrl ?? documentUrl, url ?? String.Empty); + + // the spec throws a TypeError for these, the payload related + // rejections below only return false + if (target.IsInvalid || (target.Scheme != ProtocolNames.Http && target.Scheme != ProtocolNames.Https)) + { + throw new InvalidOperationException("The beacon URL must be a valid http or https URL."); + } + + var loader = context.GetService(); + + if (loader is null || !TryExtract(data, out var body, out var contentType)) + { + return false; + } + + var options = context.GetService() ?? new BeaconOptions(); + var sender = Senders.GetValue(context, c => new BeaconSender(c)); + var origin = String.IsNullOrEmpty(documentUrl.Origin) ? "null" : documentUrl.Origin; + var beacon = new Beacon(loader, target, body, contentType, origin, document.DocumentUri); + return sender.TryEnqueue(beacon, options); + } + + private Boolean TryEnqueue(Beacon beacon, BeaconOptions options) + { + var length = beacon.Body.LongLength; + var start = false; + + lock (_queue) + { + if (length > 0 && _pending + length > options.Quota) + { + return false; + } + + _pending += length; + + if (options.MaxConcurrentRequests <= 0 || _running < options.MaxConcurrentRequests) + { + _running++; + start = true; + } + else + { + _queue.Enqueue(beacon); + } + } + + if (start) + { + _ = RunAsync(beacon); + } + + return true; + } + + private async Task RunAsync(Beacon beacon) + { + // nobody awaits a beacon, so failures are swallowed here and only + // release the quota, like a network error would in browsers + try + { + if (!beacon.RequiresPreflight || await PreflightAsync(beacon).ConfigureAwait(false)) + { + var download = beacon.Loader.FetchAsync(beacon.CreateRequest()); + var response = await download.Task.ConfigureAwait(false); + response?.Dispose(); + } + } + catch + { + } + + var next = default(Beacon); + + lock (_queue) + { + _pending -= beacon.Body.LongLength; + + if (_queue.Count > 0) + { + next = _queue.Dequeue(); + } + else + { + _running--; + } + } + + if (next is not null) + { + _ = RunAsync(next); + } + } + + private async Task PreflightAsync(Beacon beacon) + { + // the preflight goes to the requester directly as it must be + // sent without credentials, i.e., without the loader's cookies + var requester = _context.GetServices().FirstOrDefault(m => m.SupportsProtocol(beacon.Target.Scheme)); + + if (requester is null) + { + return false; + } + + var request = new Request + { + Method = HttpMethod.Options, + Address = beacon.Target, + Content = Stream.Null, + }; + + request.Headers[HeaderNames.Origin] = beacon.Origin; + request.Headers["Access-Control-Request-Method"] = "POST"; + request.Headers["Access-Control-Request-Headers"] = "content-type"; + + using (var response = await requester.RequestAsync(request, CancellationToken.None).ConfigureAwait(false)) + { + return response is not null && IsPreflightSuccessful(response, beacon.Origin); + } + } + + private static Boolean IsPreflightSuccessful(IResponse response, String origin) + { + var status = (Int32)response.StatusCode; + + if (status < 200 || status > 299) + { + return false; + } + + // beacons are sent with credentials, which rules out the + // wildcards for the allowed origin and headers + var allowedOrigin = GetHeader(response, HeaderNames.AccessControlAllowOrigin); + var allowCredentials = GetHeader(response, "Access-Control-Allow-Credentials"); + var allowedHeaders = GetHeader(response, "Access-Control-Allow-Headers") ?? String.Empty; + + return allowedOrigin == origin && + allowCredentials == "true" && + allowedHeaders.Split(',').Any(name => name.Trim().Equals("content-type", StringComparison.OrdinalIgnoreCase)); + } + + private static String GetHeader(IResponse response, String name) + { + foreach (var pair in response.Headers) + { + if (pair.Key.Equals(name, StringComparison.OrdinalIgnoreCase)) + { + return pair.Value?.Trim(); + } + } + + return null; + } + + private static Boolean IsSafelistedContentType(String contentType) + { + if (contentType is null) + { + return true; + } + + if (Encoding.UTF8.GetByteCount(contentType) > 128) + { + return false; + } + + var separator = contentType.IndexOf(';'); + var essence = (separator < 0 ? contentType : contentType.Substring(0, separator)).Trim(); + return SafelistedContentTypes.Any(type => type.Equals(essence, StringComparison.OrdinalIgnoreCase)); + } + + private static Boolean TryExtract(Object data, out Byte[] body, out String contentType) + { + body = Array.Empty(); + contentType = null; + + switch (data) + { + case null: + return true; + case String text: + body = Encoding.UTF8.GetBytes(text); + contentType = "text/plain;charset=UTF-8"; + return true; + case Stream _: + // like a ReadableStream, a stream cannot be used for a + // keepalive request as its length is unknown upfront + return false; + case IBlob blob when blob.Body is null: + contentType = String.IsNullOrEmpty(blob.Type) ? null : blob.Type; + return true; + } + + try + { + var serialized = RequestBody.Serialize(data); + + using (var buffer = new MemoryStream()) + { + serialized.Content.CopyTo(buffer); + body = buffer.ToArray(); + } + + contentType = String.IsNullOrEmpty(serialized.ContentType) ? null : serialized.ContentType; + return true; + } + catch + { + return false; + } + } + + private sealed class Beacon + { + public Beacon(IDocumentLoader loader, Url target, Byte[] body, String contentType, String origin, String referer) + { + Loader = loader; + Target = target; + Body = body; + ContentType = contentType; + Origin = origin; + Referer = referer; + + // a non-safelisted content type switches the request to the + // cors mode, which needs a preflight for other origins + RequiresPreflight = !IsSafelistedContentType(contentType) && target.Origin != origin; + } + + public IDocumentLoader Loader { get; } + + public Url Target { get; } + + public Byte[] Body { get; } + + public String ContentType { get; } + + public String Origin { get; } + + public String Referer { get; } + + public Boolean RequiresPreflight { get; } + + public DocumentRequest CreateRequest() + { + var request = new DocumentRequest(Target) + { + Method = HttpMethod.Post, + Body = Body.Length > 0 ? new MemoryStream(Body, false) : Stream.Null, + Referer = Referer, + }; + + request.Headers[HeaderNames.Origin] = Origin; + + if (ContentType is not null) + { + request.Headers[HeaderNames.ContentType] = ContentType; + } + + return request; + } + } + } +} diff --git a/src/AngleSharp.Io/Dom/Clipboard.cs b/src/AngleSharp.Io/Dom/Clipboard.cs index 9db1126..1a17291 100644 --- a/src/AngleSharp.Io/Dom/Clipboard.cs +++ b/src/AngleSharp.Io/Dom/Clipboard.cs @@ -1,6 +1,7 @@ namespace AngleSharp.Io.Dom { using AngleSharp.Attributes; + using AngleSharp.Io.PermissionsApi; using System; using System.Threading.Tasks; @@ -11,14 +12,27 @@ namespace AngleSharp.Io.Dom public sealed class Clipboard { private readonly IClipboardPlatform _platform; + private readonly Permissions _permissions; /// /// Creates a new clipboard wrapper using the provided platform implementation. /// /// The platform clipboard implementation. public Clipboard(IClipboardPlatform platform) + : this(platform, null) + { + } + + /// + /// Creates a new clipboard wrapper using the provided platform + /// implementation, which is only used when the permissions allow it. + /// + /// The platform clipboard implementation. + /// The permissions to check, or null to allow every access. + public Clipboard(IClipboardPlatform platform, Permissions permissions) { _platform = platform ?? throw new ArgumentNullException(nameof(platform)); + _permissions = permissions; } /// @@ -26,9 +40,14 @@ public Clipboard(IClipboardPlatform platform) /// /// The clipboard text. [DomName("readText")] - public Task ReadText() + public async Task ReadText() { - return _platform.ReadTextAsync(); + if (_permissions != null) + { + await _permissions.EnsureGrantedAsync(PermissionNames.ClipboardRead).ConfigureAwait(false); + } + + return await _platform.ReadTextAsync().ConfigureAwait(false); } /// @@ -37,9 +56,14 @@ public Task ReadText() /// The text to write. /// The task representing the asynchronous operation. [DomName("writeText")] - public Task WriteText(String text) + public async Task WriteText(String text) { - return _platform.WriteTextAsync(text ?? String.Empty); + if (_permissions != null) + { + await _permissions.EnsureGrantedAsync(PermissionNames.ClipboardWrite).ConfigureAwait(false); + } + + await _platform.WriteTextAsync(text ?? String.Empty).ConfigureAwait(false); } } } \ No newline at end of file diff --git a/src/AngleSharp.Io/Dom/Geolocation.cs b/src/AngleSharp.Io/Dom/Geolocation.cs index 6b7e1e7..4eccad7 100644 --- a/src/AngleSharp.Io/Dom/Geolocation.cs +++ b/src/AngleSharp.Io/Dom/Geolocation.cs @@ -1,6 +1,7 @@ namespace AngleSharp.Io.Dom { using AngleSharp.Attributes; + using AngleSharp.Io.PermissionsApi; using System; using System.Threading.Tasks; @@ -11,14 +12,27 @@ namespace AngleSharp.Io.Dom public sealed class Geolocation { private readonly IGeolocationPlatform _platform; + private readonly Permissions _permissions; /// /// Creates a new geolocation wrapper using the provided platform implementation. /// /// The platform geolocation implementation. public Geolocation(IGeolocationPlatform platform) + : this(platform, null) + { + } + + /// + /// Creates a new geolocation wrapper using the provided platform + /// implementation, which is only used when the permissions allow it. + /// + /// The platform geolocation implementation. + /// The permissions to check, or null to allow every access. + public Geolocation(IGeolocationPlatform platform, Permissions permissions) { _platform = platform ?? throw new ArgumentNullException(nameof(platform)); + _permissions = permissions; } /// @@ -39,6 +53,11 @@ public Task GetCurrentPosition() [DomName("getCurrentPosition")] public async Task GetCurrentPosition(GeolocationOptions options) { + if (_permissions != null) + { + await _permissions.EnsureGrantedAsync(PermissionNames.Geolocation).ConfigureAwait(false); + } + var reading = await _platform.GetCurrentPositionAsync(options ?? new GeolocationOptions()).ConfigureAwait(false); return GeolocationPosition.FromReading(reading ?? GeolocationReading.Empty); } diff --git a/src/AngleSharp.Io/Dom/LockManager.cs b/src/AngleSharp.Io/Dom/LockManager.cs index 06405b0..5936293 100644 --- a/src/AngleSharp.Io/Dom/LockManager.cs +++ b/src/AngleSharp.Io/Dom/LockManager.cs @@ -93,7 +93,7 @@ private async Task RequestInternal(String name, Func /// Defines a set of extensions for the navigator object. @@ -10,6 +11,28 @@ namespace AngleSharp.Io.Dom [DomExposed("Navigator")] public static class NavigatorExtensions { + private static readonly ConditionalWeakTable PermissionsByNavigator = new ConditionalWeakTable(); + + /// + /// Gets the permissions object. + /// + [DomName("permissions")] + [DomAccessor(Accessors.Getter)] + public static Permissions Permissions(this INavigator navigator) + { + var context = GetContext(navigator); + var policy = context?.GetService(); + + if (policy == null) + { + return null; + } + + // the origin is resolved per call as the active document changes + // while the navigator stays the same + return PermissionsByNavigator.GetValue(navigator, _ => new Permissions(policy, () => LockManager.GetOrigin(context.Active?.DefaultView))); + } + /// /// Gets the clipboard object. /// @@ -34,6 +57,66 @@ public static Geolocation Geolocation(this INavigator navigator) return factory?.GetGeolocation(navigator); } + /// + /// Gets the storage manager object. + /// + [DomName("storage")] + [DomAccessor(Accessors.Getter)] + public static StorageManager Storage(this INavigator navigator) + { + var context = GetContext(navigator); + var factory = context?.GetService(); + return factory?.GetStorageManager(navigator); + } + + /// + /// Gets the service worker container object. Only available in secure + /// contexts, i.e., for https or localhost documents. + /// + [DomName("serviceWorker")] + [DomAccessor(Accessors.Getter)] + public static ServiceWorkerContainer ServiceWorker(this INavigator navigator) + { + var context = GetContext(navigator); + var address = context?.Active?.Url; + + if (String.IsNullOrEmpty(address) || !ServiceWorkerContainer.IsPotentiallyTrustworthy(new AngleSharp.Dom.Url(address))) + { + return null; + } + + var factory = context.GetService(); + return factory?.GetServiceWorkerContainer(navigator); + } + + /// + /// Queues a POST request without body to the given URL. + /// + /// The navigator sending the beacon. + /// The target URL, relative to the current document. + /// True if the beacon was queued, otherwise false. + [DomName("sendBeacon")] + public static Boolean SendBeacon(this INavigator navigator, String url) => navigator.SendBeacon(url, null); + + /// + /// Queues a POST request with the given data to the given URL. The + /// request is sent in the background, i.e., its outcome is not + /// observable. + /// + /// The navigator sending the beacon. + /// The target URL, relative to the current document. + /// + /// The data to send, e.g., a string, a blob, a byte array, form data, + /// or URL search parameters. + /// + /// + /// True if the beacon was queued, otherwise false, e.g., when the data + /// exceeds the quota or is not supported. + /// + [DomName("sendBeacon")] + public static Boolean SendBeacon(this INavigator navigator, String url, Object data) => + BeaconSender.Send(GetContext(navigator), url, data); + /// /// Gets the locks object. /// @@ -52,7 +135,7 @@ public static LockManager Locks(this INavigator navigator) return String.IsNullOrEmpty(scope) ? null : new LockManager(scope); } - private static AngleSharp.IBrowsingContext GetContext(INavigator navigator) + internal static AngleSharp.IBrowsingContext GetContext(INavigator navigator) { return (navigator as Navigator)?.Context; } diff --git a/src/AngleSharp.Io/Dom/PermissionChangedEventArgs.cs b/src/AngleSharp.Io/Dom/PermissionChangedEventArgs.cs new file mode 100644 index 0000000..7859b97 --- /dev/null +++ b/src/AngleSharp.Io/Dom/PermissionChangedEventArgs.cs @@ -0,0 +1,30 @@ +namespace AngleSharp.Io.Dom; + +using System; + +/// +/// The arguments of a change of a permission policy. +/// +public sealed class PermissionChangedEventArgs : EventArgs +{ + /// + /// Creates new arguments for the given change. + /// + /// The affected origin, or empty if all origins are affected. + /// The name of the changed permission. + public PermissionChangedEventArgs(String origin, String name) + { + Origin = origin ?? String.Empty; + Name = name ?? throw new ArgumentNullException(nameof(name)); + } + + /// + /// Gets the affected origin, or empty if all origins are affected. + /// + public String Origin { get; } + + /// + /// Gets the name of the changed permission. + /// + public String Name { get; } +} diff --git a/src/AngleSharp.Io/Dom/PermissionDescriptor.cs b/src/AngleSharp.Io/Dom/PermissionDescriptor.cs new file mode 100644 index 0000000..55d1476 --- /dev/null +++ b/src/AngleSharp.Io/Dom/PermissionDescriptor.cs @@ -0,0 +1,17 @@ +namespace AngleSharp.Io.Dom; + +using AngleSharp.Attributes; +using System; + +/// +/// Describes the permission to query. +/// +[DomName("PermissionDescriptor")] +public sealed class PermissionDescriptor +{ + /// + /// Gets or sets the name of the permission, e.g., "geolocation". + /// + [DomName("name")] + public String Name { get; set; } +} diff --git a/src/AngleSharp.Io/Dom/PermissionState.cs b/src/AngleSharp.Io/Dom/PermissionState.cs new file mode 100644 index 0000000..94b5091 --- /dev/null +++ b/src/AngleSharp.Io/Dom/PermissionState.cs @@ -0,0 +1,27 @@ +namespace AngleSharp.Io.Dom; + +using AngleSharp.Attributes; + +/// +/// The states of a permission. +/// +[DomLiterals] +[DomName("PermissionState")] +public enum PermissionState +{ + /// + /// The feature can be used without asking. + /// + [DomName("granted")] + Granted, + /// + /// The feature cannot be used. + /// + [DomName("denied")] + Denied, + /// + /// Using the feature requires asking the user first. + /// + [DomName("prompt")] + Prompt, +} diff --git a/src/AngleSharp.Io/Dom/PermissionStatus.cs b/src/AngleSharp.Io/Dom/PermissionStatus.cs new file mode 100644 index 0000000..d4e9c87 --- /dev/null +++ b/src/AngleSharp.Io/Dom/PermissionStatus.cs @@ -0,0 +1,101 @@ +namespace AngleSharp.Io.Dom; + +using AngleSharp.Attributes; +using AngleSharp.Dom; +using AngleSharp.Dom.Events; +using System; + +/// +/// Represents the status of a permission for the origin it was queried +/// for. The state follows the changes of the policy. +/// +[DomName("PermissionStatus")] +public sealed class PermissionStatus : EventTarget +{ + private readonly IPermissionPolicy _policy; + private readonly String _origin; + private readonly String _name; + + private PermissionState _state; + + internal PermissionStatus(IPermissionPolicy policy, String origin, String name) + { + _policy = policy; + _origin = origin; + _name = name; + _state = policy.Query(origin, name); + Subscription.Attach(policy, this); + } + + /// + /// Gets the name of the permission. + /// + [DomName("name")] + public String Name => _name; + + /// + /// Gets the current state of the permission. + /// + [DomName("state")] + public PermissionState State => _state; + + /// + /// Adds or removes the handler for the change event. + /// + [DomName("onchange")] + public event DomEventHandler Changed + { + add { AddEventListener(EventNames.Change, value, false); } + remove { RemoveEventListener(EventNames.Change, value, false); } + } + + private void OnPolicyChanged(PermissionChangedEventArgs e) + { + var affected = String.Equals(e.Name, _name, StringComparison.Ordinal) && + (e.Origin.Length == 0 || String.Equals(e.Origin, _origin, StringComparison.Ordinal)); + + if (affected) + { + var state = _policy.Query(_origin, _name); + + if (state != _state) + { + _state = state; + Dispatch(new Event(EventNames.Change)); + } + } + } + + // the policy usually outlives the documents, so subscribing the status + // directly would keep every status ever queried alive; the weak + // subscription detaches itself once its status has been collected + private sealed class Subscription + { + private readonly IPermissionPolicy _policy; + private readonly WeakReference _status; + + private Subscription(IPermissionPolicy policy, PermissionStatus status) + { + _policy = policy; + _status = new WeakReference(status); + } + + public static void Attach(IPermissionPolicy policy, PermissionStatus status) + { + var subscription = new Subscription(policy, status); + policy.Changed += subscription.OnChanged; + } + + private void OnChanged(Object sender, PermissionChangedEventArgs e) + { + if (_status.TryGetTarget(out var status)) + { + status.OnPolicyChanged(e); + } + else + { + _policy.Changed -= OnChanged; + } + } + } +} diff --git a/src/AngleSharp.Io/Dom/Permissions.cs b/src/AngleSharp.Io/Dom/Permissions.cs new file mode 100644 index 0000000..49107e7 --- /dev/null +++ b/src/AngleSharp.Io/Dom/Permissions.cs @@ -0,0 +1,105 @@ +namespace AngleSharp.Io.Dom; + +using AngleSharp.Attributes; +using AngleSharp.Dom; +using System; +using System.Threading.Tasks; + +/// +/// Represents the Permissions API, which is also the single place where +/// other APIs check if they may use a powerful feature. +/// +[DomName("Permissions")] +public sealed class Permissions +{ + // AngleSharp's DomError has no entry for it; the string constructor + // makes it at least appear under the name used by the spec + internal static readonly String NotAllowedError = "NotAllowedError"; + + private readonly IPermissionPolicy _policy; + private readonly Func _getOrigin; + + /// + /// Creates new permissions backed by the given policy. + /// + /// The policy deciding about the permissions. + /// The callback for resolving the current origin. + public Permissions(IPermissionPolicy policy, Func getOrigin) + { + _policy = policy ?? throw new ArgumentNullException(nameof(policy)); + _getOrigin = getOrigin ?? throw new ArgumentNullException(nameof(getOrigin)); + } + + /// + /// Gets the policy deciding about the permissions. + /// + public IPermissionPolicy Policy => _policy; + + /// + /// Queries the state of the described permission. + /// + /// The descriptor naming the permission. + /// The status, which follows later changes of the state. + [DomName("query")] + public Task Query(PermissionDescriptor descriptor) + { + var name = descriptor?.Name; + + // the spec rejects with a TypeError as unknown names fail the + // conversion to the PermissionName enumeration + if (String.IsNullOrEmpty(name) || !_policy.IsSupported(name)) + { + return Task.FromException(new ArgumentException($"The permission '{name}' is not supported.", nameof(descriptor))); + } + + var status = new PermissionStatus(_policy, GetOrigin(), name); + return Task.FromResult(status); + } + + /// + /// Gets the current state of the permission for the current origin + /// without asking anyone. + /// + /// The name of the permission. + /// The current state. + public PermissionState GetState(String name) => _policy.Query(GetOrigin(), name); + + /// + /// Requests the permission for the current origin, as done by an API + /// before using the feature. + /// + /// The name of the permission. + /// The state after the request, which is never prompt. + public async Task RequestAsync(String name) + { + var origin = GetOrigin(); + var state = _policy.Query(origin, name); + + if (state == PermissionState.Prompt) + { + state = await _policy.RequestAsync(origin, name).ConfigureAwait(false); + } + + // a policy leaving it at prompt did not get an answer, which must + // not unlock the feature + return state == PermissionState.Granted ? state : PermissionState.Denied; + } + + /// + /// Requests the permission for the current origin and throws a + /// NotAllowedError if it is not granted. + /// + /// The name of the permission. + /// The task completing once the permission is granted. + public async Task EnsureGrantedAsync(String name) + { + var state = await RequestAsync(name).ConfigureAwait(false); + + if (state != PermissionState.Granted) + { + throw new DomException(NotAllowedError); + } + } + + private String GetOrigin() => _getOrigin.Invoke() ?? String.Empty; +} diff --git a/src/AngleSharp.Io/Dom/ReportingObserver.cs b/src/AngleSharp.Io/Dom/ReportingObserver.cs new file mode 100644 index 0000000..765648c --- /dev/null +++ b/src/AngleSharp.Io/Dom/ReportingObserver.cs @@ -0,0 +1,225 @@ +namespace AngleSharp.Io.Dom +{ + using AngleSharp.Attributes; + using AngleSharp.Browser; + using AngleSharp.Dom; + using AngleSharp.Io.Reporting; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Runtime.CompilerServices; + + /// + /// The callback receiving the reports of a reporting observer. + /// + /// The observed reports. + /// The observer. + [DomName("ReportingObserverCallback")] + public delegate void ReportingObserverCallback(Report[] reports, ReportingObserver observer); + + /// + /// Represents the ReportingObserver interface, which observes the + /// reports queued for a window. As in browsers, the callback is invoked + /// in a task of the context's event loop; without an event loop it is + /// invoked right away. + /// + [DomName("ReportingObserver")] + [DomExposed("Window")] + public sealed class ReportingObserver + { + private readonly IWindow _window; + private readonly ReportingObserverCallback _callback; + private readonly HashSet _types; + private readonly Boolean _buffered; + private readonly List _records; + + /// + /// Creates a new reporting observer for all report types. + /// + /// The window to observe. + /// The callback receiving the reports. + [DomConstructor] + public ReportingObserver(IWindow window, ReportingObserverCallback callback) + : this(window, callback, null) + { + } + + /// + /// Creates a new reporting observer. + /// + /// The window to observe. + /// The callback receiving the reports. + /// The options of the observer. + [DomConstructor] + public ReportingObserver(IWindow window, ReportingObserverCallback callback, ReportingObserverOptions options) + { + _window = window ?? throw new ArgumentNullException(nameof(window)); + _callback = callback ?? throw new ArgumentNullException(nameof(callback)); + _types = options?.Types is null ? null : new HashSet(options.Types, StringComparer.Ordinal); + _buffered = options?.Buffered ?? false; + _records = new List(); + } + + /// + /// Starts observing the reports of the window. With the buffered + /// option, the reports queued before are delivered right away. + /// + [DomName("observe")] + public void Observe() + { + var buffered = WindowReports.For(_window).Register(this); + + if (_buffered) + { + foreach (var report in buffered) + { + Add(report); + } + } + } + + /// + /// Stops observing the reports of the window. + /// + [DomName("disconnect")] + public void Disconnect() + { + WindowReports.For(_window).Unregister(this); + } + + /// + /// Takes the reports that have not been delivered to the callback yet. + /// + /// The pending reports. + [DomName("takeRecords")] + public Report[] TakeRecords() + { + lock (_records) + { + var records = _records.ToArray(); + _records.Clear(); + return records; + } + } + + internal void Add(Report report) + { + if (_types is not null && !_types.Contains(report.Type)) + { + return; + } + + Boolean first; + + lock (_records) + { + first = _records.Count == 0; + _records.Add(report); + } + + // like the spec, only the first report schedules the callback, + // which then receives all reports added until the task runs + if (first) + { + var loop = _window.Document?.Context?.GetService(); + + if (loop is null) + { + Notify(); + } + else + { + loop.Enqueue(Notify, TaskPriority.Normal); + } + } + } + + private void Notify() + { + var records = TakeRecords(); + + if (records.Length > 0) + { + _callback.Invoke(records, this); + } + } + } + + /// + /// Represents the options of a reporting observer. + /// + [DomName("ReportingObserverOptions")] + public sealed class ReportingObserverOptions + { + /// + /// Gets or sets the report types to observe, or null for all types. + /// + [DomName("types")] + public String[] Types { get; set; } + + /// + /// Gets or sets if reports queued before observing are delivered. + /// + [DomName("buffered")] + public Boolean Buffered { get; set; } + } + + /// + /// The report buffer and the observers of a window. + /// + sealed class WindowReports + { + // the size of the report buffer used by browsers + private const Int32 BufferSize = 100; + + private static readonly ConditionalWeakTable Instances = new ConditionalWeakTable(); + + private readonly List _buffer = new List(); + private readonly List _observers = new List(); + + public static WindowReports For(IWindow window) => Instances.GetValue(window, _ => new WindowReports()); + + public Report[] Register(ReportingObserver observer) + { + lock (_buffer) + { + if (!_observers.Contains(observer)) + { + _observers.Add(observer); + } + + return _buffer.ToArray(); + } + } + + public void Unregister(ReportingObserver observer) + { + lock (_buffer) + { + _observers.Remove(observer); + } + } + + public void Add(Report report) + { + ReportingObserver[] observers; + + lock (_buffer) + { + _buffer.Add(report); + + if (_buffer.Count > BufferSize) + { + _buffer.RemoveAt(0); + } + + observers = _observers.ToArray(); + } + + foreach (var observer in observers) + { + observer.Add(report); + } + } + } +} diff --git a/src/AngleSharp.Io/Dom/RequestBody.cs b/src/AngleSharp.Io/Dom/RequestBody.cs new file mode 100644 index 0000000..260af77 --- /dev/null +++ b/src/AngleSharp.Io/Dom/RequestBody.cs @@ -0,0 +1,79 @@ +namespace AngleSharp.Io.Dom +{ + using AngleSharp.Dom; + using AngleSharp.Html; + using System; + using System.IO; + using System.Text; + + /// + /// Serializes the body forms accepted by fetch, XMLHttpRequest, and + /// sendBeacon. + /// + static class RequestBody + { + public static SerializedBody Serialize(Object body) + { + if (body == null) + { + return SerializedBody.Empty; + } + + switch (body) + { + case Stream stream: + if (stream.CanSeek) + { + stream.Seek(0, SeekOrigin.Begin); + } + + return new SerializedBody(stream); + case Byte[] bytes: + return new SerializedBody(new MemoryStream(bytes)); + case ArraySegment segment when segment.Array != null: + return new SerializedBody(new MemoryStream(segment.Array, segment.Offset, segment.Count, false)); + case FormDataSet formDataSet: + var formDataBody = formDataSet.AsMultipart(null, Encoding.UTF8); + var formDataType = String.Concat("multipart/form-data; boundary=", formDataSet.Boundary); + return new SerializedBody(formDataBody, formDataType); + case UrlSearchParams searchParams: + var query = searchParams.ToString(); + var queryBytes = Encoding.UTF8.GetBytes(query); + return new SerializedBody(new MemoryStream(queryBytes), "application/x-www-form-urlencoded; charset=UTF-8"); + case IBlob blob: + var blobBody = blob.Body; + + if (blobBody != null) + { + if (blobBody.CanSeek) + { + blobBody.Seek(0, SeekOrigin.Begin); + } + + return new SerializedBody(blobBody, blob.Type); + } + + break; + } + + var content = body.ToString(); + var contentBytes = Encoding.UTF8.GetBytes(content); + return new SerializedBody(new MemoryStream(contentBytes)); + } + } + + readonly struct SerializedBody + { + public static readonly SerializedBody Empty = new SerializedBody(Stream.Null); + + public SerializedBody(Stream content, String contentType = null) + { + Content = content ?? Stream.Null; + ContentType = contentType; + } + + public Stream Content { get; } + + public String ContentType { get; } + } +} diff --git a/src/AngleSharp.Io/Dom/ServiceWorker.cs b/src/AngleSharp.Io/Dom/ServiceWorker.cs new file mode 100644 index 0000000..258f856 --- /dev/null +++ b/src/AngleSharp.Io/Dom/ServiceWorker.cs @@ -0,0 +1,89 @@ +namespace AngleSharp.Io.Dom +{ + using AngleSharp.Attributes; + using AngleSharp.Dom; + using AngleSharp.Dom.Events; + using System; + + /// + /// Represents a service worker. Only its lifecycle is modeled, running + /// the script is up to the . + /// + [DomName("ServiceWorker")] + public sealed class ServiceWorker : EventTarget + { + private ServiceWorkerState _state; + + internal ServiceWorker(ServiceWorkerRegistration registration, String scriptUrl, WorkerType type) + { + Registration = registration; + ScriptUrl = scriptUrl; + Type = type; + _state = ServiceWorkerState.Parsed; + } + + /// + /// Gets the URL of the worker's script. + /// + [DomName("scriptURL")] + public String ScriptUrl { get; } + + /// + /// Gets the lifecycle state of the worker. + /// + [DomName("state")] + public ServiceWorkerState State => _state; + + /// + /// Gets the type of the worker's script. + /// + public WorkerType Type { get; } + + /// + /// Gets the registration owning the worker. + /// + public ServiceWorkerRegistration Registration { get; } + + /// + /// Adds or removes the handler for the statechange event. + /// + [DomName("onstatechange")] + public event DomEventHandler StateChanged + { + add { AddEventListener(ServiceWorkerEventNames.StateChange, value, false); } + remove { RemoveEventListener(ServiceWorkerEventNames.StateChange, value, false); } + } + + /// + /// Sends a message to the worker. Messages to a redundant worker are + /// dropped. + /// + /// The message to send. + [DomName("postMessage")] + public void PostMessage(Object message) + { + if (_state != ServiceWorkerState.Redundant) + { + Registration.Registry.Platform.PostMessage(this, message); + } + } + + internal void SetState(ServiceWorkerState state) + { + if (_state != state) + { + _state = state; + Dispatch(new Event(ServiceWorkerEventNames.StateChange)); + } + } + } + + static class ServiceWorkerEventNames + { + public static readonly String StateChange = "statechange"; + + public static readonly String UpdateFound = "updatefound"; + + public static readonly String ControllerChange = "controllerchange"; + } +} diff --git a/src/AngleSharp.Io/Dom/ServiceWorkerContainer.cs b/src/AngleSharp.Io/Dom/ServiceWorkerContainer.cs new file mode 100644 index 0000000..636d607 --- /dev/null +++ b/src/AngleSharp.Io/Dom/ServiceWorkerContainer.cs @@ -0,0 +1,272 @@ +namespace AngleSharp.Io.Dom +{ + using AngleSharp.Attributes; + using AngleSharp.Dom; + using AngleSharp.Dom.Events; + using AngleSharp.Io.ServiceWorkerApi; + using System; + using System.Net; + using System.Threading.Tasks; + + /// + /// Represents the ServiceWorkerContainer API exposed on navigator.serviceWorker. + /// + [DomName("ServiceWorkerContainer")] + public sealed class ServiceWorkerContainer : EventTarget + { + private readonly ServiceWorkerRegistry _registry; + private readonly Func _getClientUrl; + + /// + /// Creates a new service worker container. + /// + /// The registry holding the registrations. + /// The callback for resolving the URL of the current document. + public ServiceWorkerContainer(ServiceWorkerRegistry registry, Func getClientUrl) + { + _registry = registry ?? throw new ArgumentNullException(nameof(registry)); + _getClientUrl = getClientUrl ?? throw new ArgumentNullException(nameof(getClientUrl)); + _registry.Track(this); + } + + /// + /// Gets the active worker of the registration matching the current + /// document, if any. Unlike browsers, the match is determined on + /// access and not when the document has been loaded. + /// + [DomName("controller")] + public ServiceWorker Controller + { + get + { + var clientUrl = _getClientUrl.Invoke(); + + if (!IsPotentiallyTrustworthy(clientUrl)) + { + return null; + } + + var worker = _registry.Match(clientUrl)?.Active; + return worker?.State == ServiceWorkerState.Activated ? worker : null; + } + } + + /// + /// Gets the task completing with the registration matching the current + /// document once it has an active worker. As in browsers, the task + /// never fails, i.e., it stays pending without such a registration. + /// + [DomName("ready")] + public Task Ready + { + get + { + var clientUrl = _getClientUrl.Invoke(); + + if (!IsPotentiallyTrustworthy(clientUrl)) + { + return new TaskCompletionSource().Task; + } + + return _registry.WaitForActiveAsync(clientUrl); + } + } + + /// + /// Adds or removes the handler for the controllerchange event. + /// + [DomName("oncontrollerchange")] + public event DomEventHandler ControllerChanged + { + add { AddEventListener(ServiceWorkerEventNames.ControllerChange, value, false); } + remove { RemoveEventListener(ServiceWorkerEventNames.ControllerChange, value, false); } + } + + /// + /// Adds or removes the handler for the message event, which receives + /// the messages of the controlling worker. + /// + [DomName("onmessage")] + public event DomEventHandler Message + { + add { AddEventListener(EventNames.Message, value, false); } + remove { RemoveEventListener(EventNames.Message, value, false); } + } + + /// + /// Registers a service worker using the default options. + /// + /// The URL of the worker's script. + /// The registration. + [DomName("register")] + public Task Register(String scriptUrl) => Register(scriptUrl, null); + + /// + /// Registers a service worker. The returned task completes after the + /// worker went through its lifecycle, i.e., it is either activated or, + /// if its installation failed, redundant. + /// + /// The URL of the worker's script. + /// The registration options. + /// The registration. + [DomName("register")] + public async Task Register(String scriptUrl, ServiceWorkerRegistrationOptions options) + { + var clientUrl = GetClientUrl(); + var script = ParseUrl(clientUrl, scriptUrl ?? String.Empty); + var scope = String.IsNullOrEmpty(options?.Scope) ? new Url(script, "./") : ParseUrl(clientUrl, options.Scope); + + if (script.Origin != clientUrl.Origin || scope.Origin != clientUrl.Origin) + { + throw new DomException(DomError.Security); + } + + // without fetching the script there is no Service-Worker-Allowed + // header, which leaves the script's directory as maximum scope + var maxScope = new Url(script, "./"); + + if (!scope.PathName.StartsWith(maxScope.PathName, StringComparison.Ordinal)) + { + throw new DomException(DomError.Security); + } + + var type = options?.Type ?? WorkerType.Classic; + var updateViaCache = options?.UpdateViaCache ?? ServiceWorkerUpdateViaCache.Imports; + return await _registry.RegisterAsync(script, scope, type, updateViaCache).ConfigureAwait(false); + } + + /// + /// Gets the registration matching the current document. + /// + /// The registration or null. + [DomName("getRegistration")] + public Task GetRegistration() => GetRegistration(null); + + /// + /// Gets the registration matching the given URL. + /// + /// The URL to match, relative to the current document. + /// The registration or null. + [DomName("getRegistration")] + public Task GetRegistration(String clientUrl) + { + var baseUrl = GetClientUrl(); + var url = String.IsNullOrEmpty(clientUrl) ? baseUrl : ParseUrl(baseUrl, clientUrl); + + if (url.Origin != baseUrl.Origin) + { + throw new DomException(DomError.Security); + } + + return Task.FromResult(_registry.Match(url)); + } + + /// + /// Gets all registrations of the current document's origin. + /// + /// The registrations. + [DomName("getRegistrations")] + public Task GetRegistrations() + { + var clientUrl = GetClientUrl(); + return Task.FromResult(_registry.GetRegistrations(clientUrl.Origin)); + } + + internal void RaiseControllerChange() => Dispatch(new Event(ServiceWorkerEventNames.ControllerChange)); + + internal void Receive(ServiceWorker source, Object message) + { + // MessageEvent only knows windows as source, hence the worker is + // only reflected in the origin + var origin = new Url(source.ScriptUrl).Origin; + Dispatch(new MessageEvent(EventNames.Message, false, false, message, origin, String.Empty, null, Array.Empty())); + } + + /// + /// Checks if the given URL belongs to a potentially trustworthy origin, + /// i.e., to a secure context as far as service workers are concerned. + /// + /// The URL to check. + /// True if the URL is potentially trustworthy, otherwise false. + internal static Boolean IsPotentiallyTrustworthy(Url url) + { + if (url is null || url.IsInvalid || !IsHttp(url)) + { + return false; + } + + if (url.Scheme == ProtocolNames.Https) + { + return true; + } + + var host = (url.HostName ?? String.Empty).Trim('[', ']'); + + if (host.Equals("localhost", StringComparison.OrdinalIgnoreCase) || host.EndsWith(".localhost", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return IPAddress.TryParse(host, out var address) && IPAddress.IsLoopback(address); + } + + private Url GetClientUrl() + { + var clientUrl = _getClientUrl.Invoke(); + + if (!IsPotentiallyTrustworthy(clientUrl)) + { + throw new DomException(DomError.Security); + } + + return clientUrl; + } + + private static Url ParseUrl(Url baseUrl, String value) + { + var url = new Url(baseUrl, value); + + // the spec rejects these with a TypeError + if (url.IsInvalid || !IsHttp(url)) + { + throw new InvalidOperationException("The URL must be a valid http or https URL."); + } + + if (url.PathName.IndexOf("%2f", StringComparison.OrdinalIgnoreCase) >= 0 || url.PathName.IndexOf("%5c", StringComparison.OrdinalIgnoreCase) >= 0) + { + throw new InvalidOperationException("The URL path must not contain an escaped slash or backslash."); + } + + url.Fragment = null; + return url; + } + + private static Boolean IsHttp(Url url) => + url.Scheme == ProtocolNames.Http || url.Scheme == ProtocolNames.Https; + } + + /// + /// Represents the options for registering a service worker. + /// + [DomName("RegistrationOptions")] + public sealed class ServiceWorkerRegistrationOptions + { + /// + /// Gets or sets the scope, relative to the current document. + /// + [DomName("scope")] + public String Scope { get; set; } + + /// + /// Gets or sets the type of the worker's script. + /// + [DomName("type")] + public WorkerType Type { get; set; } = WorkerType.Classic; + + /// + /// Gets or sets how the HTTP cache is used when checking for updates. + /// + [DomName("updateViaCache")] + public ServiceWorkerUpdateViaCache UpdateViaCache { get; set; } = ServiceWorkerUpdateViaCache.Imports; + } +} diff --git a/src/AngleSharp.Io/Dom/ServiceWorkerRegistration.cs b/src/AngleSharp.Io/Dom/ServiceWorkerRegistration.cs new file mode 100644 index 0000000..b1ef41b --- /dev/null +++ b/src/AngleSharp.Io/Dom/ServiceWorkerRegistration.cs @@ -0,0 +1,86 @@ +namespace AngleSharp.Io.Dom +{ + using AngleSharp.Attributes; + using AngleSharp.Dom; + using AngleSharp.Dom.Events; + using AngleSharp.Io.ServiceWorkerApi; + using System; + using System.Threading.Tasks; + + /// + /// Represents a service worker registration, i.e., the workers bound to + /// a scope. + /// + [DomName("ServiceWorkerRegistration")] + public sealed class ServiceWorkerRegistration : EventTarget + { + private readonly ServiceWorkerRegistry _registry; + + internal ServiceWorkerRegistration(ServiceWorkerRegistry registry, String scope) + { + _registry = registry; + Scope = scope; + } + + /// + /// Gets the scope URL of the registration. + /// + [DomName("scope")] + public String Scope { get; } + + /// + /// Gets how the HTTP cache is used when checking for updates. + /// + [DomName("updateViaCache")] + public ServiceWorkerUpdateViaCache UpdateViaCache { get; internal set; } + + /// + /// Gets the worker being installed, if any. + /// + [DomName("installing")] + public ServiceWorker Installing { get; internal set; } + + /// + /// Gets the installed worker waiting for its activation, if any. + /// + [DomName("waiting")] + public ServiceWorker Waiting { get; internal set; } + + /// + /// Gets the active worker, if any. + /// + [DomName("active")] + public ServiceWorker Active { get; internal set; } + + /// + /// Adds or removes the handler for the updatefound event. + /// + [DomName("onupdatefound")] + public event DomEventHandler UpdateFound + { + add { AddEventListener(ServiceWorkerEventNames.UpdateFound, value, false); } + remove { RemoveEventListener(ServiceWorkerEventNames.UpdateFound, value, false); } + } + + /// + /// Checks the newest worker for an update, which the platform decides. + /// An update is installed and activated before the task completes. + /// + /// The registration. + [DomName("update")] + public Task Update() => _registry.UpdateAsync(this); + + /// + /// Removes the registration. Its workers become redundant right away. + /// + /// True if the registration was removed, otherwise false. + [DomName("unregister")] + public Task Unregister() => _registry.UnregisterAsync(this); + + internal ServiceWorkerRegistry Registry => _registry; + + internal ServiceWorker NewestWorker => Installing ?? Waiting ?? Active; + + internal void RaiseUpdateFound() => Dispatch(new Event(ServiceWorkerEventNames.UpdateFound)); + } +} diff --git a/src/AngleSharp.Io/Dom/ServiceWorkerState.cs b/src/AngleSharp.Io/Dom/ServiceWorkerState.cs new file mode 100644 index 0000000..67b49d4 --- /dev/null +++ b/src/AngleSharp.Io/Dom/ServiceWorkerState.cs @@ -0,0 +1,42 @@ +namespace AngleSharp.Io.Dom; + +using AngleSharp.Attributes; + +/// +/// The lifecycle states of a service worker. +/// +[DomLiterals] +[DomName("ServiceWorkerState")] +public enum ServiceWorkerState +{ + /// + /// The worker has been created, but not installed yet. + /// + [DomName("parsed")] + Parsed, + /// + /// The worker is being installed. + /// + [DomName("installing")] + Installing, + /// + /// The worker is installed and waits for its activation. + /// + [DomName("installed")] + Installed, + /// + /// The worker is being activated. + /// + [DomName("activating")] + Activating, + /// + /// The worker is active and handles fetches. + /// + [DomName("activated")] + Activated, + /// + /// The worker failed to install or has been replaced. + /// + [DomName("redundant")] + Redundant, +} diff --git a/src/AngleSharp.Io/Dom/ServiceWorkerUpdateViaCache.cs b/src/AngleSharp.Io/Dom/ServiceWorkerUpdateViaCache.cs new file mode 100644 index 0000000..350b20a --- /dev/null +++ b/src/AngleSharp.Io/Dom/ServiceWorkerUpdateViaCache.cs @@ -0,0 +1,27 @@ +namespace AngleSharp.Io.Dom; + +using AngleSharp.Attributes; + +/// +/// The ways the HTTP cache is used when checking a service worker for updates. +/// +[DomLiterals] +[DomName("ServiceWorkerUpdateViaCache")] +public enum ServiceWorkerUpdateViaCache +{ + /// + /// The HTTP cache is used for imported scripts, but not for the main script. + /// + [DomName("imports")] + Imports, + /// + /// The HTTP cache is used for all scripts. + /// + [DomName("all")] + All, + /// + /// The HTTP cache is not used for any script. + /// + [DomName("none")] + None, +} diff --git a/src/AngleSharp.Io/Dom/StorageManager.cs b/src/AngleSharp.Io/Dom/StorageManager.cs new file mode 100644 index 0000000..1f1a2d6 --- /dev/null +++ b/src/AngleSharp.Io/Dom/StorageManager.cs @@ -0,0 +1,118 @@ +namespace AngleSharp.Io.Dom +{ + using AngleSharp.Attributes; + using AngleSharp.Io.PermissionsApi; + using System; + using System.Threading.Tasks; + + /// + /// Represents the StorageManager API. + /// + [DomName("StorageManager")] + public sealed class StorageManager + { + private readonly IStorageManagerPlatform _platform; + private readonly Func _getOrigin; + private readonly Permissions _permissions; + + /// + /// Creates a new storage manager using the provided platform implementation. + /// + /// The platform storage implementation. + /// The callback for resolving the current origin. + public StorageManager(IStorageManagerPlatform platform, Func getOrigin) + : this(platform, getOrigin, null) + { + } + + /// + /// Creates a new storage manager using the provided platform + /// implementation, which only persists the storage when the + /// permissions allow it. + /// + /// The platform storage implementation. + /// The callback for resolving the current origin. + /// The permissions to check, or null to allow every request. + public StorageManager(IStorageManagerPlatform platform, Func getOrigin, Permissions permissions) + { + _platform = platform ?? throw new ArgumentNullException(nameof(platform)); + _getOrigin = getOrigin ?? throw new ArgumentNullException(nameof(getOrigin)); + _permissions = permissions; + } + + /// + /// Estimates the usage and quota of the storage. + /// + /// The storage estimate. + [DomName("estimate")] + public async Task Estimate() + { + var origin = GetOrigin(); + var estimate = await _platform.EstimateAsync(origin).ConfigureAwait(false); + return estimate ?? new StorageEstimate(); + } + + /// + /// Requests to persist the storage, i.e., to exclude it from eviction. + /// + /// True if the storage is persisted, otherwise false. + [DomName("persist")] + public async Task Persist() + { + var origin = GetOrigin(); + + // unlike other APIs a denied permission does not reject, the + // spec just resolves with false + if (_permissions != null && await _permissions.RequestAsync(PermissionNames.PersistentStorage).ConfigureAwait(false) != PermissionState.Granted) + { + return false; + } + + return await _platform.PersistAsync(origin).ConfigureAwait(false); + } + + /// + /// Checks if the storage is persisted. + /// + /// True if the storage is persisted, otherwise false. + [DomName("persisted")] + public async Task Persisted() + { + var origin = GetOrigin(); + return await _platform.PersistedAsync(origin).ConfigureAwait(false); + } + + private String GetOrigin() + { + var origin = _getOrigin.Invoke(); + + // the spec rejects with a TypeError when no storage shelf can be + // obtained, which is the case for opaque origins + if (String.IsNullOrEmpty(origin)) + { + throw new InvalidOperationException("The storage is not available for an opaque origin."); + } + + return origin; + } + } + + /// + /// Represents a storage estimate. + /// + [DomName("StorageEstimate")] + public sealed class StorageEstimate + { + /// + /// Gets or sets the used storage in bytes. + /// + [DomName("usage")] + public UInt64 Usage { get; set; } + + /// + /// Gets or sets the available storage in bytes. + /// + [DomName("quota")] + public UInt64 Quota { get; set; } + } +} diff --git a/src/AngleSharp.Io/Dom/WindowExtensions.cs b/src/AngleSharp.Io/Dom/WindowExtensions.cs index a29f8d6..368d982 100644 --- a/src/AngleSharp.Io/Dom/WindowExtensions.cs +++ b/src/AngleSharp.Io/Dom/WindowExtensions.cs @@ -5,6 +5,7 @@ namespace AngleSharp.Io.Dom using AngleSharp.Io; using AngleSharp.Dom; using AngleSharp.Attributes; + using AngleSharp.Io.Reporting; using System; using System.Collections.Generic; using System.IO; @@ -61,6 +62,41 @@ public static ICacheStorage Caches(this IWindow window) return factory?.GetCaches(window); } + /// + /// Queues a report about the window's document, e.g., a deprecation. + /// The report is delivered to the window's reporting observers and, + /// if a reporting service is registered, queued for its sinks. + /// + /// The window the report is about. + /// The type of the report. + /// The type specific details, if any. + /// The name of the endpoint receiving the report. + /// The queued report. + public static Report QueueReport(this IWindow window, String type, ReportBody body, String destination = null) + { + if (window == null) + { + throw new ArgumentNullException(nameof(window)); + } + + // like browsers, the report omits credentials and the fragment + var url = new Url(window.Document?.Url ?? String.Empty) + { + Fragment = null, + UserName = null, + Password = null, + }; + + var report = new Report(type, url.Href, body, destination) + { + UserAgent = window.Navigator?.UserAgent ?? String.Empty, + }; + + WindowReports.For(window).Add(report); + window.Document?.Context?.GetService()?.QueueReport(report); + return report; + } + /// /// Performs a request using the configured document loader. /// @@ -111,7 +147,7 @@ public static async Task Fetch(this IWindow window, String url, Fetch private static DocumentRequest CreateRequest(IWindow window, Url url, FetchOptions options) { var requestOptions = options ?? new FetchOptions(); - var serializedBody = SerializeBody(requestOptions.Body); + var serializedBody = RequestBody.Serialize(requestOptions.Body); var request = new DocumentRequest(url) { Body = serializedBody.Content, @@ -146,55 +182,6 @@ private static HttpMethod ParseMethod(String method) return Enum.TryParse(method, true, out HttpMethod parsed) ? parsed : HttpMethod.Get; } - private static SerializedBody SerializeBody(Object body) - { - if (body == null) - { - return SerializedBody.Empty; - } - - switch (body) - { - case Stream stream: - if (stream.CanSeek) - { - stream.Seek(0, SeekOrigin.Begin); - } - - return new SerializedBody(stream); - case Byte[] bytes: - return new SerializedBody(new MemoryStream(bytes)); - case ArraySegment segment when segment.Array != null: - return new SerializedBody(new MemoryStream(segment.Array, segment.Offset, segment.Count, false)); - case FormDataSet formDataSet: - var formDataBody = formDataSet.AsMultipart(null, Encoding.UTF8); - var formDataType = String.Concat("multipart/form-data; boundary=", formDataSet.Boundary); - return new SerializedBody(formDataBody, formDataType); - case UrlSearchParams searchParams: - var query = searchParams.ToString(); - var queryBytes = Encoding.UTF8.GetBytes(query); - return new SerializedBody(new MemoryStream(queryBytes), "application/x-www-form-urlencoded; charset=UTF-8"); - case IBlob blob: - var blobBody = blob.Body; - - if (blobBody != null) - { - if (blobBody.CanSeek) - { - blobBody.Seek(0, SeekOrigin.Begin); - } - - return new SerializedBody(blobBody, blob.Type); - } - - break; - } - - var content = body.ToString(); - var contentBytes = Encoding.UTF8.GetBytes(content); - return new SerializedBody(new MemoryStream(contentBytes)); - } - private static Boolean ContainsHeader(Dictionary headers, String name) { foreach (var pair in headers) @@ -220,21 +207,6 @@ private static Boolean HasHeaderValue(Dictionary headers, String return false; } - - private readonly struct SerializedBody - { - public static readonly SerializedBody Empty = new SerializedBody(Stream.Null); - - public SerializedBody(Stream content, String contentType = null) - { - Content = content ?? Stream.Null; - ContentType = contentType; - } - - public Stream Content { get; } - - public String ContentType { get; } - } } /// diff --git a/src/AngleSharp.Io/Dom/WorkerType.cs b/src/AngleSharp.Io/Dom/WorkerType.cs new file mode 100644 index 0000000..c4bf4b3 --- /dev/null +++ b/src/AngleSharp.Io/Dom/WorkerType.cs @@ -0,0 +1,22 @@ +namespace AngleSharp.Io.Dom; + +using AngleSharp.Attributes; + +/// +/// The types of worker scripts. +/// +[DomLiterals] +[DomName("WorkerType")] +public enum WorkerType +{ + /// + /// A classic script. + /// + [DomName("classic")] + Classic, + /// + /// A module script. + /// + [DomName("module")] + Module, +} diff --git a/src/AngleSharp.Io/Dom/XmlHttpRequest.cs b/src/AngleSharp.Io/Dom/XmlHttpRequest.cs index 574c990..4b6882e 100644 --- a/src/AngleSharp.Io/Dom/XmlHttpRequest.cs +++ b/src/AngleSharp.Io/Dom/XmlHttpRequest.cs @@ -219,7 +219,7 @@ public void Send(Object body = null) { if (_readyState == RequesterState.Opened) { - var requestBody = Serialize(body); + var requestBody = RequestBody.Serialize(body); if (!String.IsNullOrEmpty(requestBody.ContentType) && !ContainsHeader(_headers, HeaderNames.ContentType)) { @@ -388,55 +388,6 @@ private IEventLoop GetEventLoop() => private IDocumentLoader GetLoader() => _window?.Document?.Context.GetService(); - private static SerializedBody Serialize(Object body) - { - if (body == null) - { - return SerializedBody.Empty; - } - - switch (body) - { - case Stream stream: - if (stream.CanSeek) - { - stream.Seek(0, SeekOrigin.Begin); - } - - return new SerializedBody(stream); - case Byte[] bytes: - return new SerializedBody(new MemoryStream(bytes)); - case ArraySegment segment when segment.Array != null: - return new SerializedBody(new MemoryStream(segment.Array, segment.Offset, segment.Count, false)); - case FormDataSet formDataSet: - var formDataBody = formDataSet.AsMultipart(null, Encoding.UTF8); - var formDataType = String.Concat("multipart/form-data; boundary=", formDataSet.Boundary); - return new SerializedBody(formDataBody, formDataType); - case UrlSearchParams searchParams: - var query = searchParams.ToString(); - var queryBytes = Encoding.UTF8.GetBytes(query); - return new SerializedBody(new MemoryStream(queryBytes), "application/x-www-form-urlencoded; charset=UTF-8"); - case IBlob blob: - var blobBody = blob.Body; - - if (blobBody != null) - { - if (blobBody.CanSeek) - { - blobBody.Seek(0, SeekOrigin.Begin); - } - - return new SerializedBody(blobBody, blob.Type); - } - - break; - } - - var content = body.ToString(); - var contentBytes = Encoding.UTF8.GetBytes(content); - return new SerializedBody(new MemoryStream(contentBytes)); - } - private static Boolean ContainsHeader(Dictionary headers, String name) { foreach (var pair in headers) @@ -450,21 +401,6 @@ private static Boolean ContainsHeader(Dictionary headers, String return false; } - private readonly struct SerializedBody - { - public static readonly SerializedBody Empty = new SerializedBody(Stream.Null); - - public SerializedBody(Stream content, String contentType = null) - { - Content = content ?? Stream.Null; - ContentType = contentType; - } - - public Stream Content { get; } - - public String ContentType { get; } - } - private void Fire(String eventName) => GetEventLoop().Enqueue(() => Dispatch(new Event(eventName))); diff --git a/src/AngleSharp.Io/Geolocation/GeolocationProviderFactory.cs b/src/AngleSharp.Io/Geolocation/GeolocationProviderFactory.cs index ad4b9aa..2b5d74c 100644 --- a/src/AngleSharp.Io/Geolocation/GeolocationProviderFactory.cs +++ b/src/AngleSharp.Io/Geolocation/GeolocationProviderFactory.cs @@ -31,7 +31,7 @@ public Geolocation GetGeolocation(INavigator navigator) return null; } - return _geolocations.GetValue(navigator, _ => new Geolocation(_platform)); + return _geolocations.GetValue(navigator, _ => new Geolocation(_platform, navigator.Permissions())); } } } \ No newline at end of file diff --git a/src/AngleSharp.Io/Interfaces/IPermissionPolicy.cs b/src/AngleSharp.Io/Interfaces/IPermissionPolicy.cs new file mode 100644 index 0000000..b4d1a37 --- /dev/null +++ b/src/AngleSharp.Io/Interfaces/IPermissionPolicy.cs @@ -0,0 +1,41 @@ +namespace AngleSharp.Io.Dom; + +using System; +using System.Threading.Tasks; + +/// +/// Represents the policy deciding about the permissions of the origins. +/// It backs navigator.permissions and is consulted by every API using a +/// powerful feature, e.g., the clipboard or the geolocation. +/// +public interface IPermissionPolicy +{ + /// + /// Raised when the state of a permission changed. + /// + event EventHandler Changed; + + /// + /// Checks if the permission with the given name is known. + /// + /// The name of the permission. + /// True if the permission is known, otherwise false. + Boolean IsSupported(String name); + + /// + /// Gets the current state of the permission without asking anyone. + /// + /// The origin using the feature, or empty for an opaque origin. + /// The name of the permission. + /// The current state. + PermissionState Query(String origin, String name); + + /// + /// Requests the permission, which resolves a prompt state to either + /// granted or denied. + /// + /// The origin using the feature, or empty for an opaque origin. + /// The name of the permission. + /// The state after the request, which is never prompt. + Task RequestAsync(String origin, String name); +} diff --git a/src/AngleSharp.Io/Interfaces/IServiceWorkerPlatform.cs b/src/AngleSharp.Io/Interfaces/IServiceWorkerPlatform.cs new file mode 100644 index 0000000..09bb2fa --- /dev/null +++ b/src/AngleSharp.Io/Interfaces/IServiceWorkerPlatform.cs @@ -0,0 +1,58 @@ +namespace AngleSharp.Io.Dom +{ + using AngleSharp.Io; + using System; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Represents the platform running the service workers, e.g., a script + /// engine or a test double. + /// + public interface IServiceWorkerPlatform + { + /// + /// Installs the given worker. A faulted task marks the installation + /// as failed and the worker becomes redundant. + /// + /// The worker to install. + /// The task representing the installation. + Task InstallAsync(ServiceWorker worker); + + /// + /// Activates the given worker. As in browsers, a faulted task does + /// not prevent the activation. + /// + /// The worker to activate. + /// The task representing the activation. + Task ActivateAsync(ServiceWorker worker); + + /// + /// Checks if the script of the given worker changed, i.e., if a new + /// worker with the same script URL and type should be installed. + /// The registration's update via cache mode tells how the script + /// should be fetched. + /// + /// The newest worker of the registration. + /// True if the script changed, otherwise false. + Task CheckForUpdateAsync(ServiceWorker worker); + + /// + /// Delivers a message posted to the given worker. Replies can be sent + /// via . + /// + /// The worker receiving the message. + /// The posted message. + void PostMessage(ServiceWorker worker, Object message); + + /// + /// Handles a request within the scope of the given active worker. The + /// request content must not be consumed when null is returned. + /// + /// The active worker controlling the request. + /// The intercepted request. + /// The token for cancelling the task. + /// The response, or null to continue with the network. + Task FetchAsync(ServiceWorker worker, Request request, CancellationToken cancel); + } +} diff --git a/src/AngleSharp.Io/Interfaces/IServiceWorkerProviderFactory.cs b/src/AngleSharp.Io/Interfaces/IServiceWorkerProviderFactory.cs new file mode 100644 index 0000000..29853f9 --- /dev/null +++ b/src/AngleSharp.Io/Interfaces/IServiceWorkerProviderFactory.cs @@ -0,0 +1,17 @@ +namespace AngleSharp.Io.Dom +{ + using AngleSharp.Browser.Dom; + + /// + /// Represents a factory for creating ServiceWorkerContainer views per navigator. + /// + public interface IServiceWorkerProviderFactory + { + /// + /// Gets the ServiceWorkerContainer object for the provided navigator. + /// + /// The navigator requesting service worker access. + /// The ServiceWorkerContainer object or null. + ServiceWorkerContainer GetServiceWorkerContainer(INavigator navigator); + } +} diff --git a/src/AngleSharp.Io/Interfaces/IStorageManagerPlatform.cs b/src/AngleSharp.Io/Interfaces/IStorageManagerPlatform.cs new file mode 100644 index 0000000..080f8e4 --- /dev/null +++ b/src/AngleSharp.Io/Interfaces/IStorageManagerPlatform.cs @@ -0,0 +1,32 @@ +namespace AngleSharp.Io.Dom +{ + using System; + using System.Threading.Tasks; + + /// + /// Represents the platform storage implementation used by the DOM storage manager. + /// + public interface IStorageManagerPlatform + { + /// + /// Estimates the usage and quota of the storage for the given origin. + /// + /// The origin owning the storage. + /// The storage estimate. + Task EstimateAsync(String origin); + + /// + /// Requests to persist the storage of the given origin. + /// + /// The origin owning the storage. + /// True if the storage is persisted after the request, otherwise false. + Task PersistAsync(String origin); + + /// + /// Checks if the storage of the given origin is persisted. + /// + /// The origin owning the storage. + /// True if the storage is persisted, otherwise false. + Task PersistedAsync(String origin); + } +} diff --git a/src/AngleSharp.Io/Interfaces/IStorageManagerProviderFactory.cs b/src/AngleSharp.Io/Interfaces/IStorageManagerProviderFactory.cs new file mode 100644 index 0000000..bed322e --- /dev/null +++ b/src/AngleSharp.Io/Interfaces/IStorageManagerProviderFactory.cs @@ -0,0 +1,17 @@ +namespace AngleSharp.Io.Dom +{ + using AngleSharp.Browser.Dom; + + /// + /// Represents a factory for creating StorageManager views per navigator. + /// + public interface IStorageManagerProviderFactory + { + /// + /// Gets the StorageManager object for the provided navigator. + /// + /// The navigator requesting storage access. + /// The StorageManager object or null. + StorageManager GetStorageManager(INavigator navigator); + } +} diff --git a/src/AngleSharp.Io/IoConfigurationExtensions.cs b/src/AngleSharp.Io/IoConfigurationExtensions.cs index c6b7128..72e8c88 100644 --- a/src/AngleSharp.Io/IoConfigurationExtensions.cs +++ b/src/AngleSharp.Io/IoConfigurationExtensions.cs @@ -8,8 +8,13 @@ namespace AngleSharp; using AngleSharp.Io.GeolocationApi; using AngleSharp.Io.Http; using AngleSharp.Io.Network; +using AngleSharp.Io.PermissionsApi; +using AngleSharp.Io.Reporting; using AngleSharp.Io.Storage; using AngleSharp.Io.ClipboardApi; +using AngleSharp.Io.ServiceWorkerApi; +using AngleSharp.Io.StorageManagerApi; +using AngleSharp.Io.WebRtc; using System; using System.IO; using System.Linq; @@ -38,14 +43,20 @@ public static IConfiguration WithIo(this IConfiguration configuration, IoOptions return configuration .WithNavigator(options.Navigator) + .WithBeacon(options.Beacon) .WithCookies(options.CookieHandler) .WithRequesters(options.HttpHandler) .WithHttpCache(options.HttpCache) + .WithReporting(options.Reporting) + .WithServiceWorkers(options.ServiceWorkerPlatform) .WithStorageProviderFactory(options.StorageFactory) .WithIndexedDbProviderFactory(options.IndexedDbFactory) .WithCacheProviderFactory(options.CacheFactory) + .WithPermissions(options.Permissions) .WithClipboard(options.ClipboardPlatform) - .WithGeolocation(options.GeolocationPlatform); + .WithGeolocation(options.GeolocationPlatform) + .WithStorageManager(options.StorageManagerPlatform) + .WithWebRtc(options.WebRtcPlatform); } #endregion @@ -75,6 +86,28 @@ public static IConfiguration WithNavigator(this IConfiguration configuration, Na #endregion + #region Beacon + + /// + /// Registers the options for sending beacons. Without them, beacons are + /// sent using the default quota. Passing null leaves the configuration + /// unchanged. + /// + /// The configuration to extend. + /// The beacon options to use. + /// The new configuration. + public static IConfiguration WithBeacon(this IConfiguration configuration, BeaconOptions options) + { + if (options is null) + { + return configuration; + } + + return configuration.WithOnly(options); + } + + #endregion + #region Download /// @@ -255,22 +288,120 @@ public static IConfiguration WithHttpCache(this IConfiguration configuration, Ht public static IConfiguration WithHttpCache(this IConfiguration configuration, IHttpCache cache) { // rebuilding the list keeps the order of the requesters, which - // decides which requester handles a protocol + // decides which requester handles a protocol; as in browsers the + // HTTP cache stays behind the service workers var services = configuration.Services.Select(service => service switch { - CachingRequester caching => new CachingRequester(caching.Inner, cache), - IRequester requester when IsHttpRequester(requester) => new CachingRequester(requester, cache), + ServiceWorkerRequester serviceWorker => new ServiceWorkerRequester(WithHttpCache(serviceWorker.Inner, cache), serviceWorker.Registry), + IRequester requester => WithHttpCache(requester, cache), _ => service, }); return new Configuration(services).WithOnly(cache); } + private static IRequester WithHttpCache(IRequester requester, IHttpCache cache) => requester switch + { + CachingRequester caching => new CachingRequester(caching.Inner, cache), + _ when IsHttpRequester(requester) => new CachingRequester(requester, cache), + _ => requester, + }; + private static Boolean IsHttpRequester(IRequester requester) => requester.SupportsProtocol(ProtocolNames.Http) || requester.SupportsProtocol(ProtocolNames.Https); #endregion + #region Reporting + + /// + /// Registers a reporting service with the default options, which keeps + /// the reports until flushed. Needs to be called after the requesters + /// have been registered to capture network errors. + /// + /// The configuration to extend. + /// The new configuration. + public static IConfiguration WithReporting(this IConfiguration configuration) => + configuration.WithReporting(new ReportingOptions()); + + /// + /// Registers a reporting service with the given options. Needs to be + /// called after the requesters have been registered to capture network + /// errors. Passing null leaves the configuration unchanged. + /// + /// The configuration to extend. + /// The options of the service. + /// The new configuration. + public static IConfiguration WithReporting(this IConfiguration configuration, ReportingOptions options) + { + if (options is null) + { + return configuration; + } + + return configuration.WithReporting(new ReportingService(options)); + } + + /// + /// Registers the given reporting service and wraps the registered HTTP(S) + /// requesters to report their network errors. The reporting sits closest + /// to the network, i.e., behind the HTTP cache and the service workers. + /// + /// The configuration to extend. + /// The service receiving the reports. + /// The new configuration. + public static IConfiguration WithReporting(this IConfiguration configuration, ReportingService service) + { + var services = configuration.Services.Select(item => item switch + { + IRequester requester => WithReporting(requester, service), + _ => item, + }); + + return new Configuration(services).WithOnly(service); + } + + private static IRequester WithReporting(IRequester requester, ReportingService service) => requester switch + { + ServiceWorkerRequester serviceWorker => new ServiceWorkerRequester(WithReporting(serviceWorker.Inner, service), serviceWorker.Registry), + CachingRequester caching => new CachingRequester(WithReporting(caching.Inner, service), caching.Cache), + ReportingRequester reporting => new ReportingRequester(reporting.Inner, service), + _ when IsHttpRequester(requester) => new ReportingRequester(requester, service), + _ => requester, + }; + + #endregion + + #region WebRTC + + /// + /// Registers the loopback WebRTC platform, which connects the peer + /// connections of the configuration with each other in-process. + /// + /// The configuration to extend. + /// The new configuration. + public static IConfiguration WithWebRtc(this IConfiguration configuration) => + configuration.WithWebRtc(new LoopbackRtcPlatform()); + + /// + /// Registers the given WebRTC platform, which provides the transport of + /// RTCPeerConnection. Passing null leaves the configuration unchanged. + /// + /// The configuration to extend. + /// The platform to use. + /// The new configuration. + public static IConfiguration WithWebRtc(this IConfiguration configuration, IRtcPlatform platform) + { + if (platform is null) + { + return configuration; + } + + return configuration.WithOnly(platform); + } + + #endregion + #region Cookies /// @@ -374,6 +505,40 @@ public static IConfiguration WithCacheProviderFactory(this IConfiguration config #endregion + #region Permissions + + /// + /// Registers a permission policy where every permission requires a + /// prompt, which is dismissed as no user can answer it. Hence, the + /// gated features cannot be used until their permissions are granted + /// via the policy found in navigator.permissions. + /// + /// The configuration to extend. + /// The new instance with the service. + public static IConfiguration WithPermissions(this IConfiguration configuration) => + configuration.WithPermissions(new PermissionPolicy()); + + /// + /// Registers the given permission policy, which backs + /// navigator.permissions and gates the clipboard, the geolocation, and + /// persisting the storage. Passing null leaves the configuration + /// unchanged, i.e., these features are not gated. + /// + /// The configuration to extend. + /// The policy deciding about the permissions. + /// The new instance with the service. + public static IConfiguration WithPermissions(this IConfiguration configuration, IPermissionPolicy policy) + { + if (policy is null) + { + return configuration; + } + + return configuration.WithOnly(policy); + } + + #endregion + #region Clipboard /// @@ -433,4 +598,89 @@ public static IConfiguration WithGeolocationProviderFactory(this IConfiguration configuration.WithOnly(factory); #endregion + + #region StorageManager + + /// + /// Registers the storage manager service using the given platform implementation. + /// + /// The configuration to extend. + /// The platform storage implementation. + /// The new instance with the service. + public static IConfiguration WithStorageManager(this IConfiguration configuration, IStorageManagerPlatform platform) + { + if (platform is not null) + { + var factory = new StorageManagerProviderFactory(platform); + return configuration.WithStorageManagerProviderFactory(factory); + } + + return configuration; + } + + /// + /// Registers a storage manager provider factory. + /// + /// The configuration to extend. + /// The storage manager provider factory to use. + /// The new instance with the service. + public static IConfiguration WithStorageManagerProviderFactory(this IConfiguration configuration, IStorageManagerProviderFactory factory) => + configuration.WithOnly(factory); + + #endregion + + #region Service Workers + + /// + /// Registers the service worker container with a platform installing and + /// activating every worker without intercepting any request. Needs to be + /// called after the requesters have been registered. + /// + /// The configuration to extend. + /// The new instance with the service. + public static IConfiguration WithServiceWorkers(this IConfiguration configuration) => + configuration.WithServiceWorkers(new ServiceWorkerPlatform()); + + /// + /// Registers the service worker container using the given platform + /// implementation. Needs to be called after the requesters have been + /// registered. Passing null leaves the configuration unchanged. + /// + /// The configuration to extend. + /// The platform running the workers. + /// The new instance with the service. + public static IConfiguration WithServiceWorkers(this IConfiguration configuration, IServiceWorkerPlatform platform) + { + if (platform is not null) + { + return configuration.WithServiceWorkers(new ServiceWorkerRegistry(platform)); + } + + return configuration; + } + + /// + /// Registers the service worker container using the given registry, which + /// is also registered as a service. The registered HTTP(S) requesters are + /// wrapped to pass requests to the active workers. Needs to be called + /// after the requesters have been registered. + /// + /// The configuration to extend. + /// The registry holding the registrations. + /// The new instance with the service. + public static IConfiguration WithServiceWorkers(this IConfiguration configuration, ServiceWorkerRegistry registry) + { + var services = configuration.Services.Select(service => service switch + { + ServiceWorkerRequester serviceWorker => new ServiceWorkerRequester(serviceWorker.Inner, registry), + IRequester requester when IsHttpRequester(requester) => new ServiceWorkerRequester(requester, registry), + _ => service, + }); + + return new Configuration(services) + .WithOnly(registry) + .WithOnly(new ServiceWorkerProviderFactory(registry)); + } + + #endregion } diff --git a/src/AngleSharp.Io/IoOptions.cs b/src/AngleSharp.Io/IoOptions.cs index 1e8f7a5..0807fd2 100644 --- a/src/AngleSharp.Io/IoOptions.cs +++ b/src/AngleSharp.Io/IoOptions.cs @@ -8,7 +8,10 @@ namespace AngleSharp.Io; using AngleSharp.Io.Dom; using AngleSharp.Io.Http; using AngleSharp.Io.IndexedDb; +using AngleSharp.Io.PermissionsApi; +using AngleSharp.Io.Reporting; using AngleSharp.Io.Storage; +using AngleSharp.Io.WebRtc; /// /// Options for setting up the Io services. @@ -56,6 +59,11 @@ public static IoOptions CreatePersistent(String rootDirectoryPath) /// public NavigatorOptions Navigator { get; set; } = new NavigatorOptions(); + /// + /// Gets or sets the options for sending beacons via navigator.sendBeacon. + /// + public BeaconOptions Beacon { get; set; } = new BeaconOptions(); + /// /// Gets or sets the handler for configuring the cookie jar. /// @@ -73,6 +81,13 @@ public static IoOptions CreatePersistent(String rootDirectoryPath) /// public HttpCacheOptions HttpCache { get; set; } = null; + /// + /// Gets or sets the options for collecting reports, e.g., about network + /// errors. By default, this is null, which means that no reports are + /// collected. + /// + public ReportingOptions Reporting { get; set; } = null; + /// /// Gets or sets the provider for configuring the web storage provider. /// @@ -88,6 +103,14 @@ public static IoOptions CreatePersistent(String rootDirectoryPath) /// public ICacheProviderFactory CacheFactory { get; set; } = CacheProviderFactory.CreateTemporary(); + /// + /// Gets or sets the policy deciding about the permissions of powerful + /// features, e.g., a . By default, this is + /// null, which means that navigator.permissions is not available and + /// the features are not gated. + /// + public IPermissionPolicy Permissions { get; set; } = null; + /// /// Gets or sets the provider for configuring the clipboard platform. /// By default, this is null, which means that the clipboard API will not be available. @@ -99,4 +122,23 @@ public static IoOptions CreatePersistent(String rootDirectoryPath) /// By default, this is null, which means that the geolocation API will not be available. /// public IGeolocationPlatform GeolocationPlatform { get; set; } = null; + + /// + /// Gets or sets the provider for configuring the storage manager platform. + /// By default, this is null, which means that navigator.storage will not be available. + /// + public IStorageManagerPlatform StorageManagerPlatform { get; set; } = null; + + /// + /// Gets or sets the platform running the service workers. + /// By default, this is null, which means that navigator.serviceWorker will not be available. + /// + public IServiceWorkerPlatform ServiceWorkerPlatform { get; set; } = null; + + /// + /// Gets or sets the platform providing the transport of RTCPeerConnection, + /// e.g., a . By default, this is null, + /// which means that RTCPeerConnection cannot be constructed. + /// + public IRtcPlatform WebRtcPlatform { get; set; } = null; } diff --git a/src/AngleSharp.Io/Network/HttpClientRequester.cs b/src/AngleSharp.Io/Network/HttpClientRequester.cs index a33cc00..558d37b 100644 --- a/src/AngleSharp.Io/Network/HttpClientRequester.cs +++ b/src/AngleSharp.Io/Network/HttpClientRequester.cs @@ -8,6 +8,7 @@ namespace AngleSharp.Io.Network using System.Threading; using System.Threading.Tasks; using AngleSharp.Dom; + using AngleSharp.Io.Reporting; using HttpMethod = System.Net.Http.HttpMethod; /// @@ -135,14 +136,18 @@ protected override async Task PerformRequestAsync(Request request, Ca return response; } - catch (Exception) + catch (Exception ex) { - // create a response to avoid failing (#28) - return new DefaultResponse + // create a response to avoid failing (#28); the cause is kept + // for reporting the network error + var response = new DefaultResponse { Address = Url.Convert(request.Address), StatusCode = 0 }; + + NetworkFailures.Attach(response, ex); + return response; } } diff --git a/src/AngleSharp.Io/Permissions/PermissionNames.cs b/src/AngleSharp.Io/Permissions/PermissionNames.cs new file mode 100644 index 0000000..86f98e9 --- /dev/null +++ b/src/AngleSharp.Io/Permissions/PermissionNames.cs @@ -0,0 +1,43 @@ +namespace AngleSharp.Io.PermissionsApi; + +using System; + +/// +/// The names of the permissions known by default. +/// +public static class PermissionNames +{ + /// + /// Reading the position via navigator.geolocation. + /// + public static readonly String Geolocation = "geolocation"; + + /// + /// Showing notifications. + /// + public static readonly String Notifications = "notifications"; + + /// + /// Reading from the clipboard via navigator.clipboard. + /// + public static readonly String ClipboardRead = "clipboard-read"; + + /// + /// Writing to the clipboard via navigator.clipboard. + /// + public static readonly String ClipboardWrite = "clipboard-write"; + + /// + /// Excluding the storage from eviction via navigator.storage.persist(). + /// + public static readonly String PersistentStorage = "persistent-storage"; + + internal static readonly String[] All = new[] + { + Geolocation, + Notifications, + ClipboardRead, + ClipboardWrite, + PersistentStorage, + }; +} diff --git a/src/AngleSharp.Io/Permissions/PermissionPolicy.cs b/src/AngleSharp.Io/Permissions/PermissionPolicy.cs new file mode 100644 index 0000000..536bf43 --- /dev/null +++ b/src/AngleSharp.Io/Permissions/PermissionPolicy.cs @@ -0,0 +1,226 @@ +namespace AngleSharp.Io.PermissionsApi; + +using AngleSharp.Dom; +using AngleSharp.Io.Dom; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +/// +/// Answers a prompt for a permission in place of the user. +/// +/// The origin requesting the permission, or empty for an opaque origin. +/// The name of the permission. +/// The answer; prompt counts as a dismissed prompt. +public delegate Task PermissionPromptHandler(String origin, String name); + +/// +/// Represents the default permission policy keeping the states in memory. +/// The state of a permission is taken from the origin's entry, then from +/// the entry for all origins, and finally from the default state. +/// +public sealed class PermissionPolicy : IPermissionPolicy +{ + private readonly Object _sync; + private readonly HashSet _names; + private readonly Dictionary _allOrigins; + private readonly Dictionary> _origins; + + /// + /// Creates a new permission policy where every permission requires a + /// prompt. Without a such a prompt is + /// treated as dismissed, i.e., the feature cannot be used. + /// + public PermissionPolicy() + : this(PermissionState.Prompt) + { + } + + /// + /// Creates a new permission policy using the given default state. + /// + /// The state of permissions without an entry. + public PermissionPolicy(PermissionState defaultState) + { + _sync = new Object(); + _names = new HashSet(PermissionNames.All, StringComparer.Ordinal); + _allOrigins = new Dictionary(StringComparer.Ordinal); + _origins = new Dictionary>(StringComparer.Ordinal); + DefaultState = defaultState; + } + + /// + public event EventHandler Changed; + + /// + /// Gets the state of permissions without an entry. + /// + public PermissionState DefaultState { get; } + + /// + /// Gets or sets the handler answering prompts. An answer other than + /// prompt is remembered for the origin. By default, this is null, + /// which means that prompts are dismissed. + /// + public PermissionPromptHandler PromptHandler { get; set; } + + /// + /// Makes the permission with the given name known, e.g., for a feature + /// provided by another library. + /// + /// The name of the permission. + public void Register(String name) + { + if (String.IsNullOrEmpty(name)) + { + throw new ArgumentException("The name of the permission is required.", nameof(name)); + } + + lock (_sync) + { + _names.Add(name); + } + } + + /// + public Boolean IsSupported(String name) + { + if (String.IsNullOrEmpty(name)) + { + return false; + } + + lock (_sync) + { + return _names.Contains(name); + } + } + + /// + /// Sets the state of the permission for all origins. Entries of + /// single origins take precedence. + /// + /// The name of the permission. + /// The new state. + public void Set(String name, PermissionState state) + { + lock (_sync) + { + EnsureSupported(name); + _allOrigins[name] = state; + } + + Changed?.Invoke(this, new PermissionChangedEventArgs(String.Empty, name)); + } + + /// + /// Sets the state of the permission for the given origin. + /// + /// The origin, e.g., "https://example.com". + /// The name of the permission. + /// The new state. + public void Set(String origin, String name, PermissionState state) + { + var key = NormalizeOrigin(origin); + + if (key.Length == 0) + { + throw new ArgumentException("An origin is required.", nameof(origin)); + } + + lock (_sync) + { + EnsureSupported(name); + + if (!_origins.TryGetValue(key, out var states)) + { + states = new Dictionary(StringComparer.Ordinal); + _origins[key] = states; + } + + states[name] = state; + } + + Changed?.Invoke(this, new PermissionChangedEventArgs(key, name)); + } + + /// + public PermissionState Query(String origin, String name) + { + var key = NormalizeOrigin(origin); + + lock (_sync) + { + if (_origins.TryGetValue(key, out var states) && states.TryGetValue(name, out var state)) + { + return state; + } + + if (_allOrigins.TryGetValue(name, out state)) + { + return state; + } + + return DefaultState; + } + } + + /// + public async Task RequestAsync(String origin, String name) + { + var state = Query(origin, name); + + if (state != PermissionState.Prompt) + { + return state; + } + + var handler = PromptHandler; + + if (handler is null) + { + return PermissionState.Denied; + } + + var answer = await handler.Invoke(NormalizeOrigin(origin), name).ConfigureAwait(false); + + if (answer == PermissionState.Prompt) + { + return PermissionState.Denied; + } + + // opaque origins cannot be told apart, so their answers are not + // remembered and every use prompts again + if (NormalizeOrigin(origin).Length > 0) + { + Set(origin, name, answer); + } + + return answer; + } + + private void EnsureSupported(String name) + { + if (String.IsNullOrEmpty(name) || !_names.Contains(name)) + { + throw new ArgumentException($"The permission '{name}' is not registered.", nameof(name)); + } + } + + private static String NormalizeOrigin(String origin) + { + if (String.IsNullOrEmpty(origin)) + { + return String.Empty; + } + + var url = new Url(origin); + + if (url.IsInvalid || url.IsRelative) + { + return origin; + } + + return url.Origin ?? origin; + } +} diff --git a/src/AngleSharp.Io/Reporting/EndpointReportSink.cs b/src/AngleSharp.Io/Reporting/EndpointReportSink.cs new file mode 100644 index 0000000..ae8e0bb --- /dev/null +++ b/src/AngleSharp.Io/Reporting/EndpointReportSink.cs @@ -0,0 +1,75 @@ +namespace AngleSharp.Io.Reporting +{ + using AngleSharp.Dom; + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Delivers reports to their endpoints, i.e., posts them as + /// application/reports+json to the URL of their destination. Reports + /// without a known endpoint URL are skipped. + /// + public sealed class EndpointReportSink : IReportSink + { + private readonly IRequester _requester; + + /// + /// Creates a new endpoint sink using the given requester. The + /// requester should not be wrapped for reporting, as the delivery of + /// reports must not cause new reports. + /// + /// The requester for posting the reports. + public EndpointReportSink(IRequester requester) + { + _requester = requester ?? throw new ArgumentNullException(nameof(requester)); + } + + internal Func Clock { get; set; } = () => DateTimeOffset.UtcNow; + + /// + /// Posts the reports grouped by their endpoint URL. A failed delivery + /// throws after all endpoints have been tried. + /// + /// The reports to deliver. + /// The token for cancelling the task. + /// The task representing the delivery. + public async Task SendAsync(IReadOnlyList reports, CancellationToken cancel) + { + var failed = new List(); + var now = Clock.Invoke(); + + foreach (var group in reports.Where(m => !String.IsNullOrEmpty(m.EndpointUrl)).GroupBy(m => m.EndpointUrl, StringComparer.Ordinal)) + { + var payload = ReportJson.Serialize(group.Select(m => m.ToDeliveryDictionary(now)).ToArray()); + var request = new Request + { + Method = HttpMethod.Post, + Address = new Url(group.Key), + Content = new MemoryStream(Encoding.UTF8.GetBytes(payload), false), + }; + + request.Headers[HeaderNames.ContentType] = "application/reports+json"; + + using (var response = await _requester.RequestAsync(request, cancel).ConfigureAwait(false)) + { + var status = (Int32)(response?.StatusCode ?? 0); + + if (status < 200 || status > 299) + { + failed.Add(group.Key); + } + } + } + + if (failed.Count > 0) + { + throw new InvalidOperationException("Delivering reports failed for: " + String.Join(", ", failed)); + } + } + } +} diff --git a/src/AngleSharp.Io/Reporting/IReportSink.cs b/src/AngleSharp.Io/Reporting/IReportSink.cs new file mode 100644 index 0000000..732884e --- /dev/null +++ b/src/AngleSharp.Io/Reporting/IReportSink.cs @@ -0,0 +1,21 @@ +namespace AngleSharp.Io.Reporting +{ + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Represents a backend receiving flushed reports, e.g., for storing or + /// forwarding them. + /// + public interface IReportSink + { + /// + /// Receives the given reports. + /// + /// The reports in the order they were queued. + /// The token for cancelling the task. + /// The task representing the operation. + Task SendAsync(IReadOnlyList reports, CancellationToken cancel); + } +} diff --git a/src/AngleSharp.Io/Reporting/MemoryReportSink.cs b/src/AngleSharp.Io/Reporting/MemoryReportSink.cs new file mode 100644 index 0000000..449bc11 --- /dev/null +++ b/src/AngleSharp.Io/Reporting/MemoryReportSink.cs @@ -0,0 +1,50 @@ +namespace AngleSharp.Io.Reporting +{ + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Keeps the received reports in memory, e.g., for tests. + /// + public sealed class MemoryReportSink : IReportSink + { + private readonly List _reports = new List(); + + /// + /// Gets a snapshot of the received reports. + /// + public IReadOnlyList Reports + { + get + { + lock (_reports) + { + return _reports.ToArray(); + } + } + } + + /// + /// Removes all received reports. + /// + public void Clear() + { + lock (_reports) + { + _reports.Clear(); + } + } + + /// + public Task SendAsync(IReadOnlyList reports, CancellationToken cancel) + { + lock (_reports) + { + _reports.AddRange(reports); + } + + return Task.CompletedTask; + } + } +} diff --git a/src/AngleSharp.Io/Reporting/NetworkErrorReportBody.cs b/src/AngleSharp.Io/Reporting/NetworkErrorReportBody.cs new file mode 100644 index 0000000..89a1375 --- /dev/null +++ b/src/AngleSharp.Io/Reporting/NetworkErrorReportBody.cs @@ -0,0 +1,75 @@ +namespace AngleSharp.Io.Reporting +{ + using System; + using System.Collections.Generic; + + /// + /// Represents the body of a network-error report, following the shape + /// defined by Network Error Logging. + /// + public sealed class NetworkErrorReportBody : ReportBody + { + /// + /// Gets or sets the sampling rate applied to the report. + /// + public Double SamplingFraction { get; set; } = 1.0; + + /// + /// Gets or sets the time in milliseconds between the start of the + /// request and its completion or failure. + /// + public Int64 ElapsedTime { get; set; } + + /// + /// Gets or sets the phase in which the error occurred, i.e., dns, + /// connection, or application. + /// + public String Phase { get; set; } = String.Empty; + + /// + /// Gets or sets the type of the error, e.g., dns.name_not_resolved, + /// tcp.refused, or http.error. + /// + public String Type { get; set; } = String.Empty; + + /// + /// Gets or sets the IP address of the server, if known. + /// + public String ServerIp { get; set; } = String.Empty; + + /// + /// Gets or sets the network protocol, if known. + /// + public String Protocol { get; set; } = String.Empty; + + /// + /// Gets or sets the referrer of the request. + /// + public String Referrer { get; set; } = String.Empty; + + /// + /// Gets or sets the method of the request. + /// + public String Method { get; set; } = String.Empty; + + /// + /// Gets or sets the status code of the response, or 0 if there is + /// none. + /// + public Int32 StatusCode { get; set; } + + /// + public override IReadOnlyDictionary ToDictionary() => new Dictionary(StringComparer.Ordinal) + { + ["sampling_fraction"] = SamplingFraction, + ["elapsed_time"] = ElapsedTime, + ["phase"] = Phase, + ["type"] = Type, + ["server_ip"] = ServerIp, + ["protocol"] = Protocol, + ["referrer"] = Referrer, + ["method"] = Method, + ["status_code"] = StatusCode, + }; + } +} diff --git a/src/AngleSharp.Io/Reporting/NetworkFailures.cs b/src/AngleSharp.Io/Reporting/NetworkFailures.cs new file mode 100644 index 0000000..f78cc19 --- /dev/null +++ b/src/AngleSharp.Io/Reporting/NetworkFailures.cs @@ -0,0 +1,26 @@ +namespace AngleSharp.Io.Reporting +{ + using System; + using System.Runtime.CompilerServices; + + /// + /// Keeps the exception behind a response that requesters create instead + /// of failing, which allows reporting the cause of the network error. + /// + static class NetworkFailures + { + private static readonly ConditionalWeakTable Failures = new ConditionalWeakTable(); + + public static void Attach(IResponse response, Exception error) + { + if (response is not null && error is not null) + { + Failures.Remove(response); + Failures.Add(response, error); + } + } + + public static Exception Get(IResponse response) => + response is not null && Failures.TryGetValue(response, out var error) ? error : null; + } +} diff --git a/src/AngleSharp.Io/Reporting/Report.cs b/src/AngleSharp.Io/Reporting/Report.cs new file mode 100644 index 0000000..d33c8e0 --- /dev/null +++ b/src/AngleSharp.Io/Reporting/Report.cs @@ -0,0 +1,108 @@ +namespace AngleSharp.Io.Reporting +{ + using AngleSharp.Attributes; + using System; + using System.Collections.Generic; + + /// + /// Represents a report, e.g., about a network error or a deprecation. + /// + [DomName("Report")] + public sealed class Report + { + /// + /// The endpoint used when no destination is given. + /// + public static readonly String DefaultDestination = "default"; + + /// + /// Creates a new report. + /// + /// The type of the report, e.g., network-error. + /// The URL the report is about. + /// The type specific details, if any. + /// The name of the endpoint receiving the report. + public Report(String type, String url, ReportBody body, String destination = null) + { + if (String.IsNullOrEmpty(type)) + { + throw new ArgumentException("The type of a report must not be empty.", nameof(type)); + } + + Type = type; + Url = url ?? String.Empty; + Body = body; + Destination = String.IsNullOrEmpty(destination) ? DefaultDestination : destination; + } + + /// + /// Gets the type of the report. + /// + [DomName("type")] + public String Type { get; } + + /// + /// Gets the URL the report is about. + /// + [DomName("url")] + public String Url { get; } + + /// + /// Gets the type specific details, if any. + /// + [DomName("body")] + public ReportBody Body { get; } + + /// + /// Gets the name of the endpoint receiving the report. + /// + public String Destination { get; } + + /// + /// Gets the URL of the destination endpoint, if it is known when the + /// report is queued. + /// + public String EndpointUrl { get; internal set; } + + /// + /// Gets the user agent string of the report's origin, if known. + /// + public String UserAgent { get; internal set; } = String.Empty; + + /// + /// Gets the time the report was queued. + /// + public DateTimeOffset Timestamp { get; internal set; } + + /// + /// Serializes the report to JSON with the members type, url, and body. + /// + /// The JSON representation. + [DomName("toJSON")] + public String ToJson() => ReportJson.Serialize(ToDictionary()); + + /// + /// Serializes the report to JSON in the format used for delivering it + /// to an endpoint, i.e., including the age and the user agent. + /// + /// The time of the delivery. + /// The JSON representation. + public String ToDeliveryJson(DateTimeOffset now) => ReportJson.Serialize(ToDeliveryDictionary(now)); + + internal IReadOnlyDictionary ToDictionary() => new Dictionary(StringComparer.Ordinal) + { + ["type"] = Type, + ["url"] = Url, + ["body"] = Body?.ToDictionary(), + }; + + internal IReadOnlyDictionary ToDeliveryDictionary(DateTimeOffset now) => new Dictionary(StringComparer.Ordinal) + { + ["age"] = Math.Max(0L, (Int64)(now - Timestamp).TotalMilliseconds), + ["type"] = Type, + ["url"] = Url, + ["user_agent"] = UserAgent ?? String.Empty, + ["body"] = Body?.ToDictionary(), + }; + } +} diff --git a/src/AngleSharp.Io/Reporting/ReportBody.cs b/src/AngleSharp.Io/Reporting/ReportBody.cs new file mode 100644 index 0000000..d8640cd --- /dev/null +++ b/src/AngleSharp.Io/Reporting/ReportBody.cs @@ -0,0 +1,54 @@ +namespace AngleSharp.Io.Reporting +{ + using AngleSharp.Attributes; + using System; + using System.Collections.Generic; + + /// + /// Represents the type specific details of a report. + /// + [DomName("ReportBody")] + public abstract class ReportBody + { + /// + /// Gets the members of the body as they appear in its JSON form. + /// Values may be strings, numbers, booleans, null, lists, or nested + /// dictionaries. + /// + /// The members of the body. + public abstract IReadOnlyDictionary ToDictionary(); + + /// + /// Serializes the body to JSON. + /// + /// The JSON representation. + [DomName("toJSON")] + public String ToJson() => ReportJson.Serialize(ToDictionary()); + } + + /// + /// Represents a report body with arbitrary members, e.g., for custom + /// report types. + /// + public sealed class GenericReportBody : ReportBody + { + private readonly Dictionary _members; + + /// + /// Creates a new report body with the given members. + /// + /// The members of the body. + public GenericReportBody(IEnumerable> members) + { + _members = new Dictionary(StringComparer.Ordinal); + + foreach (var member in members ?? Array.Empty>()) + { + _members[member.Key] = member.Value; + } + } + + /// + public override IReadOnlyDictionary ToDictionary() => _members; + } +} diff --git a/src/AngleSharp.Io/Reporting/ReportEventArgs.cs b/src/AngleSharp.Io/Reporting/ReportEventArgs.cs new file mode 100644 index 0000000..a829e05 --- /dev/null +++ b/src/AngleSharp.Io/Reporting/ReportEventArgs.cs @@ -0,0 +1,24 @@ +namespace AngleSharp.Io.Reporting +{ + using System; + + /// + /// The arguments for the event raised when a report is queued. + /// + public sealed class ReportEventArgs : EventArgs + { + /// + /// Creates new arguments for the given report. + /// + /// The queued report. + public ReportEventArgs(Report report) + { + Report = report; + } + + /// + /// Gets the queued report. + /// + public Report Report { get; } + } +} diff --git a/src/AngleSharp.Io/Reporting/ReportFilter.cs b/src/AngleSharp.Io/Reporting/ReportFilter.cs new file mode 100644 index 0000000..628630d --- /dev/null +++ b/src/AngleSharp.Io/Reporting/ReportFilter.cs @@ -0,0 +1,56 @@ +namespace AngleSharp.Io.Reporting +{ + using System; + using System.Collections.Generic; + + /// + /// Selects reports by their type and destination endpoint. An empty set + /// matches every report. + /// + public sealed class ReportFilter + { + /// + /// Gets the report types to match, e.g., network-error. + /// + public ISet Types { get; } = new HashSet(StringComparer.Ordinal); + + /// + /// Gets the names of the destination endpoints to match. + /// + public ISet Destinations { get; } = new HashSet(StringComparer.Ordinal); + + /// + /// Creates a filter matching the given report types. + /// + /// The report types to match. + /// The new filter. + public static ReportFilter ForTypes(params String[] types) + { + var filter = new ReportFilter(); + filter.Types.UnionWith(types ?? Array.Empty()); + return filter; + } + + /// + /// Creates a filter matching the given destination endpoints. + /// + /// The names of the endpoints to match. + /// The new filter. + public static ReportFilter ForDestinations(params String[] destinations) + { + var filter = new ReportFilter(); + filter.Destinations.UnionWith(destinations ?? Array.Empty()); + return filter; + } + + /// + /// Checks if the given report matches the filter. + /// + /// The report to check. + /// True if the report matches, otherwise false. + public Boolean Matches(Report report) => + report is not null && + (Types.Count == 0 || Types.Contains(report.Type)) && + (Destinations.Count == 0 || Destinations.Contains(report.Destination)); + } +} diff --git a/src/AngleSharp.Io/Reporting/ReportJson.cs b/src/AngleSharp.Io/Reporting/ReportJson.cs new file mode 100644 index 0000000..718ab50 --- /dev/null +++ b/src/AngleSharp.Io/Reporting/ReportJson.cs @@ -0,0 +1,166 @@ +namespace AngleSharp.Io.Reporting +{ + using System; + using System.Collections; + using System.Collections.Generic; + using System.Globalization; + using System.Linq; + using System.Text; + + /// + /// A minimal JSON writer for reports. Members are written in ordinal + /// order, which keeps the output deterministic. + /// + static class ReportJson + { + public static String Serialize(Object value) + { + var sb = new StringBuilder(); + Write(sb, value); + return sb.ToString(); + } + + private static void Write(StringBuilder sb, Object value) + { + switch (value) + { + case null: + sb.Append("null"); + break; + case String text: + WriteString(sb, text); + break; + case Boolean flag: + sb.Append(flag ? "true" : "false"); + break; + case Double number when Double.IsNaN(number) || Double.IsInfinity(number): + sb.Append("null"); + break; + case Single number when Single.IsNaN(number) || Single.IsInfinity(number): + sb.Append("null"); + break; + case Double number: + sb.Append(number.ToString("R", CultureInfo.InvariantCulture)); + break; + case Single number: + sb.Append(number.ToString("R", CultureInfo.InvariantCulture)); + break; + case Byte _: + case SByte _: + case Int16 _: + case UInt16 _: + case Int32 _: + case UInt32 _: + case Int64 _: + case UInt64 _: + case Decimal _: + sb.Append(Convert.ToString(value, CultureInfo.InvariantCulture)); + break; + case ReportBody body: + Write(sb, body.ToDictionary()); + break; + case Report report: + Write(sb, report.ToDictionary()); + break; + case IReadOnlyDictionary members: + WriteObject(sb, members); + break; + case IDictionary members: + WriteObject(sb, members); + break; + case IEnumerable items: + WriteArray(sb, items); + break; + default: + WriteString(sb, Convert.ToString(value, CultureInfo.InvariantCulture)); + break; + } + } + + private static void WriteObject(StringBuilder sb, IEnumerable> members) + { + var first = true; + sb.Append('{'); + + foreach (var member in members.OrderBy(m => m.Key, StringComparer.Ordinal)) + { + if (!first) + { + sb.Append(','); + } + + first = false; + WriteString(sb, member.Key); + sb.Append(':'); + Write(sb, member.Value); + } + + sb.Append('}'); + } + + private static void WriteArray(StringBuilder sb, IEnumerable items) + { + var first = true; + sb.Append('['); + + foreach (var item in items) + { + if (!first) + { + sb.Append(','); + } + + first = false; + Write(sb, item); + } + + sb.Append(']'); + } + + private static void WriteString(StringBuilder sb, String text) + { + sb.Append('"'); + + foreach (var c in text) + { + switch (c) + { + case '"': + sb.Append("\\\""); + break; + case '\\': + sb.Append("\\\\"); + break; + case '\n': + sb.Append("\\n"); + break; + case '\r': + sb.Append("\\r"); + break; + case '\t': + sb.Append("\\t"); + break; + case '\b': + sb.Append("\\b"); + break; + case '\f': + sb.Append("\\f"); + break; + default: + if (c < ' ') + { + sb.Append("\\u").Append(((Int32)c).ToString("x4", CultureInfo.InvariantCulture)); + } + else + { + sb.Append(c); + } + + break; + } + } + + sb.Append('"'); + } + } +} diff --git a/src/AngleSharp.Io/Reporting/ReportTypes.cs b/src/AngleSharp.Io/Reporting/ReportTypes.cs new file mode 100644 index 0000000..a6b0e5b --- /dev/null +++ b/src/AngleSharp.Io/Reporting/ReportTypes.cs @@ -0,0 +1,34 @@ +namespace AngleSharp.Io.Reporting; + +using System; + +/// +/// The report types defined by the Reporting API and related specs. +/// +public static class ReportTypes +{ + /// + /// A network error, see Network Error Logging. + /// + public static readonly String NetworkError = "network-error"; + + /// + /// The use of a deprecated API. + /// + public static readonly String Deprecation = "deprecation"; + + /// + /// A request of the document that was denied by the user agent. + /// + public static readonly String Intervention = "intervention"; + + /// + /// A crash of the document. + /// + public static readonly String Crash = "crash"; + + /// + /// A violation of the content security policy. + /// + public static readonly String CspViolation = "csp-violation"; +} diff --git a/src/AngleSharp.Io/Reporting/ReportingOptions.cs b/src/AngleSharp.Io/Reporting/ReportingOptions.cs new file mode 100644 index 0000000..52f8e40 --- /dev/null +++ b/src/AngleSharp.Io/Reporting/ReportingOptions.cs @@ -0,0 +1,125 @@ +namespace AngleSharp.Io.Reporting +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Options for collecting and delivering reports. + /// + public sealed class ReportingOptions + { + /// + /// Gets or sets the filter deciding which reports are queued. By + /// default, this is null, which means that all reports are queued. + /// + public ReportFilter Filter { get; set; } + + /// + /// Gets or sets if failed requests, i.e., requests without response, + /// are reported as network-error. By default, this is true. + /// + public Boolean CaptureNetworkErrors { get; set; } = true; + + /// + /// Gets or sets if responses with a status code of 400 or above are + /// reported as network-error of type http.error. By default, this is + /// true. + /// + public Boolean CaptureHttpErrors { get; set; } = true; + + /// + /// Gets or sets the name of the endpoint receiving network-error + /// reports. By default, this is "default". + /// + public String NetworkErrorDestination { get; set; } = Report.DefaultDestination; + + /// + /// Gets the endpoints, i.e., names mapped to URLs, used when the + /// origin of a report did not announce the endpoint via the + /// Reporting-Endpoints header. + /// + public IDictionary Endpoints { get; } = new Dictionary(StringComparer.Ordinal); + + /// + /// Gets or sets the maximum number of queued reports. The oldest + /// reports are dropped when the limit is reached. + /// + public Int32 MaxQueueSize { get; set; } = 1000; + + /// + /// Gets or sets the interval for flushing the queued reports + /// automatically. By default, this is null, which means that reports + /// are only flushed explicitly. + /// + public TimeSpan? FlushInterval { get; set; } + + /// + /// Gets the sinks receiving the flushed reports. + /// + public IList Sinks { get; } = new List(); + + /// + /// Adds a sink receiving the flushed reports matching the filter. + /// + /// The sink to add. + /// The filter for the sink, or null for all reports. + /// The options. + public ReportingOptions AddSink(IReportSink sink, ReportFilter filter = null) + { + Sinks.Add(new ReportSinkRegistration(sink, filter)); + return this; + } + + /// + /// Adds a callback receiving the flushed reports matching the filter. + /// + /// The callback to add. + /// The filter for the callback, or null for all reports. + /// The options. + public ReportingOptions AddSink(Func, CancellationToken, Task> sink, ReportFilter filter = null) => + AddSink(new DelegateReportSink(sink ?? throw new ArgumentNullException(nameof(sink))), filter); + + internal Func Clock { get; set; } = () => DateTimeOffset.UtcNow; + + private sealed class DelegateReportSink : IReportSink + { + private readonly Func, CancellationToken, Task> _sink; + + public DelegateReportSink(Func, CancellationToken, Task> sink) + { + _sink = sink; + } + + public Task SendAsync(IReadOnlyList reports, CancellationToken cancel) => _sink.Invoke(reports, cancel); + } + } + + /// + /// Represents a sink together with the filter for its reports. + /// + public sealed class ReportSinkRegistration + { + /// + /// Creates a new registration. + /// + /// The sink receiving the reports. + /// The filter for the reports, or null for all reports. + public ReportSinkRegistration(IReportSink sink, ReportFilter filter = null) + { + Sink = sink ?? throw new ArgumentNullException(nameof(sink)); + Filter = filter; + } + + /// + /// Gets the sink receiving the reports. + /// + public IReportSink Sink { get; } + + /// + /// Gets the filter for the reports, or null for all reports. + /// + public ReportFilter Filter { get; } + } +} diff --git a/src/AngleSharp.Io/Reporting/ReportingRequester.cs b/src/AngleSharp.Io/Reporting/ReportingRequester.cs new file mode 100644 index 0000000..8eb3090 --- /dev/null +++ b/src/AngleSharp.Io/Reporting/ReportingRequester.cs @@ -0,0 +1,199 @@ +namespace AngleSharp.Io.Reporting +{ + using AngleSharp.Dom; + using System; + using System.Diagnostics; + using System.Net.Sockets; + using System.Security.Authentication; + using System.Threading; + using System.Threading.Tasks; + + /// + /// A requester reporting network errors of the wrapped requester and + /// learning the endpoints announced via the Reporting-Endpoints header. + /// + public class ReportingRequester : BaseRequester + { + #region Fields + + private readonly IRequester _inner; + private readonly ReportingService _service; + + #endregion + + #region ctor + + /// + /// Creates a new reporting requester. + /// + /// The requester used for network requests. + /// The service receiving the reports. + public ReportingRequester(IRequester inner, ReportingService service) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + _service = service ?? throw new ArgumentNullException(nameof(service)); + } + + #endregion + + #region Properties + + /// + /// Gets the wrapped requester. + /// + public IRequester Inner => _inner; + + /// + /// Gets the service receiving the reports. + /// + public ReportingService Service => _service; + + #endregion + + #region Methods + + /// + /// Checks if the given protocol is supported by the wrapped requester. + /// + /// The protocol to check for, e.g., http. + /// True if the protocol is supported, otherwise false. + public override Boolean SupportsProtocol(String protocol) => _inner.SupportsProtocol(protocol); + + /// + /// Performs the request using the wrapped requester. + /// + /// The options to consider. + /// The token for cancelling the task. + /// The task that will eventually give the response data. + protected override async Task PerformRequestAsync(Request request, CancellationToken cancel) + { + var stopwatch = Stopwatch.StartNew(); + IResponse response; + + try + { + response = await _inner.RequestAsync(request, cancel).ConfigureAwait(false); + } + catch (Exception ex) + { + ReportFailure(request, ex, cancel, stopwatch.ElapsedMilliseconds); + throw; + } + + var status = (Int32)(response?.StatusCode ?? 0); + + if (status == 0) + { + ReportFailure(request, NetworkFailures.Get(response), cancel, stopwatch.ElapsedMilliseconds); + } + else + { + if (response.Headers.TryGetValue("Reporting-Endpoints", out var endpoints)) + { + _service.LearnEndpoints(response.Address ?? request.Address, endpoints); + } + + if (status >= 400 && _service.Options.CaptureHttpErrors) + { + Report(request, "application", "http.error", status, stopwatch.ElapsedMilliseconds); + } + } + + return response; + } + + #endregion + + #region Helpers + + private void ReportFailure(Request request, Exception error, CancellationToken cancel, Int64 elapsed) + { + if (_service.Options.CaptureNetworkErrors) + { + var (phase, type) = Classify(error, cancel); + Report(request, phase, type, 0, elapsed); + } + } + + private void Report(Request request, String phase, String type, Int32 status, Int64 elapsed) + { + // like NEL, the report omits credentials and the fragment + var url = new Url(request.Address) + { + Fragment = null, + UserName = null, + Password = null, + }; + + var body = new NetworkErrorReportBody + { + ElapsedTime = elapsed, + Phase = phase, + Type = type, + Referrer = GetHeader(request, HeaderNames.Referer), + Method = request.Method.ToString().ToUpperInvariant(), + StatusCode = status, + }; + + var report = new Report(ReportTypes.NetworkError, url.Href, body, _service.Options.NetworkErrorDestination) + { + UserAgent = GetHeader(request, HeaderNames.UserAgent), + }; + + _service.QueueReport(report); + } + + private static String GetHeader(Request request, String name) + { + foreach (var pair in request.Headers) + { + if (pair.Key.Equals(name, StringComparison.OrdinalIgnoreCase)) + { + return pair.Value ?? String.Empty; + } + } + + return String.Empty; + } + + internal static (String Phase, String Type) Classify(Exception error, CancellationToken cancel) + { + if (error is OperationCanceledException) + { + return cancel.IsCancellationRequested ? ("application", "abandoned") : ("connection", "tcp.timed_out"); + } + + for (var current = error; current is not null; current = current.InnerException) + { + switch (current) + { + case SocketException socket: + return ClassifySocketError(socket.SocketErrorCode); + case AuthenticationException _: + return ("connection", "tls.failed"); + case TimeoutException _: + return ("connection", "tcp.timed_out"); + } + } + + return ("application", "unknown"); + } + + private static (String Phase, String Type) ClassifySocketError(SocketError error) => error switch + { + SocketError.HostNotFound => ("dns", "dns.name_not_resolved"), + SocketError.NoData => ("dns", "dns.name_not_resolved"), + SocketError.TryAgain => ("dns", "dns.failed"), + SocketError.ConnectionRefused => ("connection", "tcp.refused"), + SocketError.ConnectionReset => ("connection", "tcp.reset"), + SocketError.ConnectionAborted => ("connection", "tcp.aborted"), + SocketError.TimedOut => ("connection", "tcp.timed_out"), + SocketError.HostUnreachable => ("connection", "tcp.address_unreachable"), + SocketError.NetworkUnreachable => ("connection", "tcp.address_unreachable"), + SocketError.AddressNotAvailable => ("connection", "tcp.address_invalid"), + _ => ("connection", "tcp.failed"), + }; + + #endregion + } +} diff --git a/src/AngleSharp.Io/Reporting/ReportingService.cs b/src/AngleSharp.Io/Reporting/ReportingService.cs new file mode 100644 index 0000000..8f3c443 --- /dev/null +++ b/src/AngleSharp.Io/Reporting/ReportingService.cs @@ -0,0 +1,228 @@ +namespace AngleSharp.Io.Reporting +{ + using AngleSharp.Dom; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Collects reports and delivers them to the configured sinks when + /// flushed. + /// + public sealed class ReportingService : IDisposable + { + private readonly ReportingOptions _options; + private readonly List _queue; + private readonly Dictionary> _endpointsByOrigin; + private readonly Timer _timer; + + /// + /// Creates a new reporting service. + /// + /// The options to use. + public ReportingService(ReportingOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _queue = new List(); + _endpointsByOrigin = new Dictionary>(StringComparer.Ordinal); + + if (options.FlushInterval is TimeSpan interval && interval > TimeSpan.Zero) + { + _timer = new Timer(_ => FlushInBackground(), null, interval, interval); + } + } + + /// + /// Raised after a report has been queued. + /// + public event EventHandler ReportQueued; + + /// + /// Gets the used options. + /// + public ReportingOptions Options => _options; + + /// + /// Queues the given report for delivery, unless it is excluded by the + /// filter of the options. + /// + /// The report to queue. + /// True if the report was queued, otherwise false. + public Boolean QueueReport(Report report) + { + if (report is null) + { + throw new ArgumentNullException(nameof(report)); + } + + if (_options.Filter is not null && !_options.Filter.Matches(report)) + { + return false; + } + + report.Timestamp = _options.Clock.Invoke(); + report.EndpointUrl = ResolveEndpoint(report.Url, report.Destination); + + lock (_queue) + { + _queue.Add(report); + + if (_options.MaxQueueSize > 0 && _queue.Count > _options.MaxQueueSize) + { + _queue.RemoveRange(0, _queue.Count - _options.MaxQueueSize); + } + } + + ReportQueued?.Invoke(this, new ReportEventArgs(report)); + return true; + } + + /// + /// Gets the queued reports matching the given filter. + /// + /// The filter to apply, or null for all reports. + /// The reports in the order they were queued. + public IReadOnlyList GetReports(ReportFilter filter = null) + { + lock (_queue) + { + return _queue.Where(m => filter is null || filter.Matches(m)).ToArray(); + } + } + + /// + /// Removes all queued reports and hands them to the sinks whose filter + /// they match. The reports are removed even if a sink fails; the + /// failure is rethrown after all sinks have been called. + /// + /// The token for cancelling the task. + /// The number of flushed reports. + public async Task FlushAsync(CancellationToken cancel = default) + { + Report[] reports; + + lock (_queue) + { + reports = _queue.ToArray(); + _queue.Clear(); + } + + var errors = new List(); + + foreach (var registration in _options.Sinks.ToArray()) + { + var selected = reports.Where(m => registration.Filter is null || registration.Filter.Matches(m)).ToArray(); + + if (selected.Length > 0) + { + try + { + await registration.Sink.SendAsync(selected, cancel).ConfigureAwait(false); + } + catch (Exception ex) + { + errors.Add(ex); + } + } + } + + if (errors.Count == 1) + { + throw errors[0]; + } + else if (errors.Count > 1) + { + throw new AggregateException(errors); + } + + return reports.Length; + } + + /// + /// Stops flushing automatically. + /// + public void Dispose() + { + _timer?.Dispose(); + } + + internal void LearnEndpoints(Url address, String header) + { + var origin = address?.Origin; + + if (String.IsNullOrEmpty(origin) || header is null) + { + return; + } + + var endpoints = new Dictionary(StringComparer.Ordinal); + + foreach (var member in SplitMembers(header)) + { + var separator = member.IndexOf('='); + + if (separator > 0) + { + var name = member.Substring(0, separator).Trim(); + var value = member.Substring(separator + 1).Trim().Trim('"'); + var url = new Url(address, value); + + if (name.Length > 0 && !url.IsInvalid && (url.Scheme == ProtocolNames.Http || url.Scheme == ProtocolNames.Https)) + { + endpoints[name] = url.Href; + } + } + } + + lock (_endpointsByOrigin) + { + _endpointsByOrigin[origin] = endpoints; + } + } + + internal String ResolveEndpoint(String reportUrl, String destination) + { + var origin = String.IsNullOrEmpty(reportUrl) ? null : new Url(reportUrl).Origin; + + lock (_endpointsByOrigin) + { + if (origin is not null && _endpointsByOrigin.TryGetValue(origin, out var endpoints) && endpoints.TryGetValue(destination, out var learned)) + { + return learned; + } + } + + return _options.Endpoints.TryGetValue(destination, out var configured) ? configured : null; + } + + private void FlushInBackground() + { + // nobody observes the automatic flushes, failing sinks have to + // wait for the next interval or an explicit flush + FlushAsync().ContinueWith(task => _ = task.Exception, TaskContinuationOptions.OnlyOnFaulted); + } + + private static IEnumerable SplitMembers(String header) + { + var start = 0; + var quoted = false; + + for (var i = 0; i < header.Length; i++) + { + if (header[i] == '"') + { + quoted = !quoted; + } + else if (header[i] == ',' && !quoted) + { + yield return header.Substring(start, i - start); + start = i + 1; + } + } + + yield return header.Substring(start); + } + } +} diff --git a/src/AngleSharp.Io/ServiceWorker/ServiceWorkerPlatform.cs b/src/AngleSharp.Io/ServiceWorker/ServiceWorkerPlatform.cs new file mode 100644 index 0000000..68ffdf8 --- /dev/null +++ b/src/AngleSharp.Io/ServiceWorker/ServiceWorkerPlatform.cs @@ -0,0 +1,59 @@ +namespace AngleSharp.Io.ServiceWorkerApi +{ + using AngleSharp.Io.Dom; + using System; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Represents a service worker platform delegating to optional callbacks. + /// Without callbacks, workers install and activate successfully, scripts + /// never change, messages are dropped, and all requests continue with + /// the network. + /// + public sealed class ServiceWorkerPlatform : IServiceWorkerPlatform + { + /// + /// Gets or sets the callback invoked when a worker is installed. + /// + public Func Install { get; set; } + + /// + /// Gets or sets the callback invoked when a worker is activated. + /// + public Func Activate { get; set; } + + /// + /// Gets or sets the callback deciding if the script of a worker + /// changed. Returning true installs a new worker. + /// + public Func> CheckForUpdate { get; set; } + + /// + /// Gets or sets the callback invoked for messages posted to a worker. + /// + public Action Message { get; set; } + + /// + /// Gets or sets the callback invoked for requests within the scope of + /// an active worker. Returning null continues with the network. + /// + public Func> Fetch { get; set; } + + /// + public Task InstallAsync(ServiceWorker worker) => Install?.Invoke(worker) ?? Task.CompletedTask; + + /// + public Task ActivateAsync(ServiceWorker worker) => Activate?.Invoke(worker) ?? Task.CompletedTask; + + /// + public Task CheckForUpdateAsync(ServiceWorker worker) => CheckForUpdate?.Invoke(worker) ?? Task.FromResult(false); + + /// + public void PostMessage(ServiceWorker worker, Object message) => Message?.Invoke(worker, message); + + /// + public Task FetchAsync(ServiceWorker worker, Request request, CancellationToken cancel) => + Fetch?.Invoke(worker, request, cancel) ?? Task.FromResult(null); + } +} diff --git a/src/AngleSharp.Io/ServiceWorker/ServiceWorkerProviderFactory.cs b/src/AngleSharp.Io/ServiceWorker/ServiceWorkerProviderFactory.cs new file mode 100644 index 0000000..7beec81 --- /dev/null +++ b/src/AngleSharp.Io/ServiceWorker/ServiceWorkerProviderFactory.cs @@ -0,0 +1,46 @@ +namespace AngleSharp.Io.ServiceWorkerApi +{ + using AngleSharp.Browser.Dom; + using AngleSharp.Dom; + using AngleSharp.Io.Dom; + using System; + using System.Runtime.CompilerServices; + + /// + /// Represents the default ServiceWorkerContainer provider factory. + /// + public sealed class ServiceWorkerProviderFactory : IServiceWorkerProviderFactory + { + private readonly ServiceWorkerRegistry _registry; + private readonly ConditionalWeakTable _containers; + + /// + /// Creates a new ServiceWorkerContainer provider factory. + /// + /// The registry holding the registrations. + public ServiceWorkerProviderFactory(ServiceWorkerRegistry registry) + { + _registry = registry ?? throw new ArgumentNullException(nameof(registry)); + _containers = new ConditionalWeakTable(); + } + + /// + public ServiceWorkerContainer GetServiceWorkerContainer(INavigator navigator) + { + if (navigator == null) + { + return null; + } + + // the client URL is resolved per call as the active document + // changes while the navigator stays the same + return _containers.GetValue(navigator, _ => new ServiceWorkerContainer(_registry, () => GetClientUrl(navigator))); + } + + private static Url GetClientUrl(INavigator navigator) + { + var address = NavigatorExtensions.GetContext(navigator)?.Active?.Url; + return String.IsNullOrEmpty(address) ? null : new Url(address); + } + } +} diff --git a/src/AngleSharp.Io/ServiceWorker/ServiceWorkerRegistry.cs b/src/AngleSharp.Io/ServiceWorker/ServiceWorkerRegistry.cs new file mode 100644 index 0000000..60868a9 --- /dev/null +++ b/src/AngleSharp.Io/ServiceWorker/ServiceWorkerRegistry.cs @@ -0,0 +1,390 @@ +namespace AngleSharp.Io.ServiceWorkerApi +{ + using AngleSharp.Dom; + using AngleSharp.Io.Dom; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Keeps the service worker registrations in memory, keyed by their + /// scope URL, and runs the lifecycle of their workers. + /// + public sealed class ServiceWorkerRegistry + { + private readonly IServiceWorkerPlatform _platform; + private readonly Dictionary _registrations; + private readonly List _readyWaiters; + private readonly List> _containers; + private readonly SemaphoreSlim _jobs; + + /// + /// Creates a new registry using the given platform. + /// + /// The platform running the workers. + public ServiceWorkerRegistry(IServiceWorkerPlatform platform) + { + _platform = platform ?? throw new ArgumentNullException(nameof(platform)); + _registrations = new Dictionary(StringComparer.Ordinal); + _readyWaiters = new List(); + _containers = new List>(); + _jobs = new SemaphoreSlim(1, 1); + } + + /// + /// Gets the platform running the workers. + /// + public IServiceWorkerPlatform Platform => _platform; + + /// + /// Registers the given script for the given scope. A registration + /// with the same script and type is returned as is, unless the update + /// via cache mode changed, which triggers an update check. Otherwise + /// a new worker is installed and activated right away, replacing the + /// active one. + /// + /// The absolute URL of the script. + /// The absolute URL of the scope. + /// The type of the script. + /// How the HTTP cache is used when checking for updates. + /// The registration. + public async Task RegisterAsync(Url scriptUrl, Url scopeUrl, WorkerType type = WorkerType.Classic, ServiceWorkerUpdateViaCache updateViaCache = ServiceWorkerUpdateViaCache.Imports) + { + if (scriptUrl is null) + { + throw new ArgumentNullException(nameof(scriptUrl)); + } + + if (scopeUrl is null) + { + throw new ArgumentNullException(nameof(scopeUrl)); + } + + // jobs are serialized like the spec's job queue, which keeps the + // lifecycle transitions of concurrent registrations consistent + await _jobs.WaitAsync().ConfigureAwait(false); + + try + { + var scope = scopeUrl.Href; + var script = scriptUrl.Href; + var registration = Find(scope); + var newest = registration?.NewestWorker; + + if (newest?.ScriptUrl == script && newest.Type == type) + { + if (registration.UpdateViaCache != updateViaCache) + { + registration.UpdateViaCache = updateViaCache; + await UpdateIfChangedAsync(registration, newest).ConfigureAwait(false); + } + + return registration; + } + + if (registration is null) + { + registration = new ServiceWorkerRegistration(this, scope); + + lock (_registrations) + { + _registrations[scope] = registration; + } + } + + registration.UpdateViaCache = updateViaCache; + await InstallAsync(registration, new ServiceWorker(registration, script, type)).ConfigureAwait(false); + return registration; + } + finally + { + _jobs.Release(); + } + } + + /// + /// Gets the registration whose scope is the longest prefix of the + /// given URL. + /// + /// The URL to match. + /// The registration or null. + public ServiceWorkerRegistration Match(Url url) + { + if (url is null) + { + return null; + } + + var href = url.Href; + var match = default(ServiceWorkerRegistration); + + lock (_registrations) + { + foreach (var registration in _registrations.Values) + { + if (href.StartsWith(registration.Scope, StringComparison.Ordinal) && (match is null || registration.Scope.Length > match.Scope.Length)) + { + match = registration; + } + } + } + + return match; + } + + /// + /// Gets the registrations of the given origin. + /// + /// The origin of the registrations. + /// The registrations ordered by their scope. + public ServiceWorkerRegistration[] GetRegistrations(String origin) + { + lock (_registrations) + { + return _registrations.Values + .Where(registration => new Url(registration.Scope).Origin == origin) + .OrderBy(registration => registration.Scope, StringComparer.Ordinal) + .ToArray(); + } + } + + /// + /// Sends a message from the given worker to the containers of the + /// documents it controls, which receive it as message event. + /// + /// The worker sending the message. + /// The message to send. + /// The number of containers that received the message. + public Int32 PostMessageToClients(ServiceWorker source, Object message) + { + if (source is null) + { + throw new ArgumentNullException(nameof(source)); + } + + var clients = GetContainers().Where(container => container.Controller == source).ToArray(); + + foreach (var client in clients) + { + client.Receive(source, message); + } + + return clients.Length; + } + + internal void Track(ServiceWorkerContainer container) + { + lock (_containers) + { + _containers.RemoveAll(reference => !reference.TryGetTarget(out _)); + _containers.Add(new WeakReference(container)); + } + } + + internal Task WaitForActiveAsync(Url clientUrl) + { + lock (_readyWaiters) + { + var registration = Match(clientUrl); + + if (registration?.Active?.State == ServiceWorkerState.Activated) + { + return Task.FromResult(registration); + } + + var waiter = new ReadyWaiter(clientUrl); + _readyWaiters.Add(waiter); + return waiter.Completion.Task; + } + } + + internal async Task UpdateAsync(ServiceWorkerRegistration registration) + { + await _jobs.WaitAsync().ConfigureAwait(false); + + try + { + var newest = registration.NewestWorker; + + if (newest is null || Find(registration.Scope) != registration) + { + throw new DomException(DomError.InvalidState); + } + + await UpdateIfChangedAsync(registration, newest).ConfigureAwait(false); + return registration; + } + finally + { + _jobs.Release(); + } + } + + internal async Task UnregisterAsync(ServiceWorkerRegistration registration) + { + await _jobs.WaitAsync().ConfigureAwait(false); + + try + { + if (!Remove(registration)) + { + return false; + } + + // browsers keep the active worker until no client uses it, + // without tracking clients it is dropped right away + foreach (var worker in new[] { registration.Installing, registration.Waiting, registration.Active }) + { + worker?.SetState(ServiceWorkerState.Redundant); + } + + registration.Installing = null; + registration.Waiting = null; + registration.Active = null; + return true; + } + finally + { + _jobs.Release(); + } + } + + private async Task UpdateIfChangedAsync(ServiceWorkerRegistration registration, ServiceWorker newest) + { + // the platform stands in for fetching the script and comparing + // it byte by byte with the one of the newest worker + if (await _platform.CheckForUpdateAsync(newest).ConfigureAwait(false)) + { + await InstallAsync(registration, new ServiceWorker(registration, newest.ScriptUrl, newest.Type)).ConfigureAwait(false); + } + } + + private async Task InstallAsync(ServiceWorkerRegistration registration, ServiceWorker worker) + { + registration.Installing = worker; + worker.SetState(ServiceWorkerState.Installing); + registration.RaiseUpdateFound(); + + try + { + await _platform.InstallAsync(worker).ConfigureAwait(false); + } + catch + { + registration.Installing = null; + worker.SetState(ServiceWorkerState.Redundant); + + if (registration.NewestWorker is null) + { + Remove(registration); + } + + return; + } + + registration.Installing = null; + registration.Waiting = worker; + worker.SetState(ServiceWorkerState.Installed); + + // without tracking clients there is nobody to wait for, hence the + // worker is activated right away as if skipWaiting() was called + var previous = registration.Active; + registration.Waiting = null; + registration.Active = worker; + previous?.SetState(ServiceWorkerState.Redundant); + worker.SetState(ServiceWorkerState.Activating); + + try + { + await _platform.ActivateAsync(worker).ConfigureAwait(false); + } + catch + { + // a failing activate event does not prevent the activation + } + + worker.SetState(ServiceWorkerState.Activated); + NotifyActivated(registration, worker); + } + + private void NotifyActivated(ServiceWorkerRegistration registration, ServiceWorker worker) + { + var ready = new List(); + + lock (_readyWaiters) + { + foreach (var waiter in _readyWaiters) + { + if (Match(waiter.ClientUrl) == registration) + { + ready.Add(waiter); + } + } + + _readyWaiters.RemoveAll(ready.Contains); + } + + foreach (var waiter in ready) + { + waiter.Completion.TrySetResult(registration); + } + + foreach (var container in GetContainers()) + { + if (container.Controller == worker) + { + container.RaiseControllerChange(); + } + } + } + + private ServiceWorkerContainer[] GetContainers() + { + lock (_containers) + { + var containers = new List(); + + foreach (var reference in _containers) + { + if (reference.TryGetTarget(out var container)) + { + containers.Add(container); + } + } + + return containers.ToArray(); + } + } + + private ServiceWorkerRegistration Find(String scope) + { + lock (_registrations) + { + return _registrations.TryGetValue(scope, out var registration) ? registration : null; + } + } + + private Boolean Remove(ServiceWorkerRegistration registration) + { + lock (_registrations) + { + return _registrations.TryGetValue(registration.Scope, out var current) && current == registration && _registrations.Remove(registration.Scope); + } + } + + private sealed class ReadyWaiter + { + public ReadyWaiter(Url clientUrl) + { + ClientUrl = clientUrl; + Completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public Url ClientUrl { get; } + + public TaskCompletionSource Completion { get; } + } + } +} diff --git a/src/AngleSharp.Io/ServiceWorker/ServiceWorkerRequester.cs b/src/AngleSharp.Io/ServiceWorker/ServiceWorkerRequester.cs new file mode 100644 index 0000000..6571e28 --- /dev/null +++ b/src/AngleSharp.Io/ServiceWorker/ServiceWorkerRequester.cs @@ -0,0 +1,88 @@ +namespace AngleSharp.Io.ServiceWorkerApi +{ + using AngleSharp.Io.Dom; + using System; + using System.Threading; + using System.Threading.Tasks; + + /// + /// A requester passing HTTP(S) requests within the scope of an active + /// service worker to the platform before delegating to the wrapped + /// requester. + /// + public class ServiceWorkerRequester : BaseRequester + { + #region Fields + + private readonly IRequester _inner; + private readonly ServiceWorkerRegistry _registry; + + #endregion + + #region ctor + + /// + /// Creates a new service worker requester. + /// + /// The requester used for network requests. + /// The registry holding the registrations. + public ServiceWorkerRequester(IRequester inner, ServiceWorkerRegistry registry) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + _registry = registry ?? throw new ArgumentNullException(nameof(registry)); + } + + #endregion + + #region Properties + + /// + /// Gets the wrapped requester. + /// + public IRequester Inner => _inner; + + /// + /// Gets the used registry. + /// + public ServiceWorkerRegistry Registry => _registry; + + #endregion + + #region Methods + + /// + /// Checks if the given protocol is supported by the wrapped requester. + /// + /// The protocol to check for, e.g., http. + /// True if the protocol is supported, otherwise false. + public override Boolean SupportsProtocol(String protocol) => _inner.SupportsProtocol(protocol); + + /// + /// Performs the request using the matching service worker, if any. + /// + /// The options to consider. + /// The token for cancelling the task. + /// The task that will eventually give the response data. + protected override async Task PerformRequestAsync(Request request, CancellationToken cancel) + { + // browsers route a client's requests to the worker controlling + // the client; requests carry no client here, so they are matched + // by their own URL like navigations are + var worker = _registry.Match(request.Address)?.Active; + + if (worker?.State == ServiceWorkerState.Activated) + { + var response = await _registry.Platform.FetchAsync(worker, request, cancel).ConfigureAwait(false); + + if (response is not null) + { + return response; + } + } + + return await _inner.RequestAsync(request, cancel).ConfigureAwait(false); + } + + #endregion + } +} diff --git a/src/AngleSharp.Io/StorageManager/MemoryStorageManagerPlatform.cs b/src/AngleSharp.Io/StorageManager/MemoryStorageManagerPlatform.cs new file mode 100644 index 0000000..09a2b0f --- /dev/null +++ b/src/AngleSharp.Io/StorageManager/MemoryStorageManagerPlatform.cs @@ -0,0 +1,73 @@ +namespace AngleSharp.Io.StorageManagerApi +{ + using AngleSharp.Io.Dom; + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + + /// + /// Represents a storage manager platform with fixed values and the + /// persistence state kept in memory, e.g., for deterministic tests. + /// + public sealed class MemoryStorageManagerPlatform : IStorageManagerPlatform + { + private readonly HashSet _persisted; + + /// + /// Creates a new memory storage manager platform. + /// + public MemoryStorageManagerPlatform() + { + _persisted = new HashSet(StringComparer.Ordinal); + } + + /// + /// Gets or sets the reported quota in bytes. + /// + public UInt64 Quota { get; set; } + + /// + /// Gets or sets the reported usage in bytes. + /// + public UInt64 Usage { get; set; } + + /// + /// Gets or sets if requests to persist the storage are granted. By + /// default, they are granted. + /// + public Boolean AllowPersistence { get; set; } = true; + + /// + public Task EstimateAsync(String origin) + { + return Task.FromResult(new StorageEstimate + { + Usage = Usage, + Quota = Quota, + }); + } + + /// + public Task PersistAsync(String origin) + { + lock (_persisted) + { + if (AllowPersistence) + { + _persisted.Add(origin); + } + + return Task.FromResult(_persisted.Contains(origin)); + } + } + + /// + public Task PersistedAsync(String origin) + { + lock (_persisted) + { + return Task.FromResult(_persisted.Contains(origin)); + } + } + } +} diff --git a/src/AngleSharp.Io/StorageManager/StorageManagerProviderFactory.cs b/src/AngleSharp.Io/StorageManager/StorageManagerProviderFactory.cs new file mode 100644 index 0000000..72d22ea --- /dev/null +++ b/src/AngleSharp.Io/StorageManager/StorageManagerProviderFactory.cs @@ -0,0 +1,45 @@ +namespace AngleSharp.Io.StorageManagerApi +{ + using AngleSharp.Browser.Dom; + using AngleSharp.Io.Dom; + using System; + using System.Runtime.CompilerServices; + + /// + /// Represents the default StorageManager provider factory. + /// + public sealed class StorageManagerProviderFactory : IStorageManagerProviderFactory + { + private readonly IStorageManagerPlatform _platform; + private readonly ConditionalWeakTable _managers; + + /// + /// Creates a new StorageManager provider factory. + /// + /// The platform storage implementation. + public StorageManagerProviderFactory(IStorageManagerPlatform platform) + { + _platform = platform ?? throw new ArgumentNullException(nameof(platform)); + _managers = new ConditionalWeakTable(); + } + + /// + public StorageManager GetStorageManager(INavigator navigator) + { + if (navigator == null) + { + return null; + } + + // the origin is resolved per call as the active document changes + // while the navigator stays the same + return _managers.GetValue(navigator, _ => new StorageManager(_platform, () => GetOrigin(navigator), navigator.Permissions())); + } + + private static String GetOrigin(INavigator navigator) + { + var window = NavigatorExtensions.GetContext(navigator)?.Active?.DefaultView; + return LockManager.GetOrigin(window); + } + } +} diff --git a/src/AngleSharp.Io/WebRtc/IRtcPlatform.cs b/src/AngleSharp.Io/WebRtc/IRtcPlatform.cs new file mode 100644 index 0000000..72ecce3 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/IRtcPlatform.cs @@ -0,0 +1,205 @@ +namespace AngleSharp.Io.WebRtc +{ + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + + /// + /// Represents the platform providing the transport behind peer + /// connections, e.g., a WebRTC stack or a test double. + /// + public interface IRtcPlatform + { + /// + /// Creates the transport of a new peer connection. + /// + /// The connection receiving the transport's events. + /// The configuration of the connection. + /// The transport of the connection. + IRtcPeer CreatePeer(IRtcPeerHost host, RtcConfiguration configuration); + } + + /// + /// Represents the transport of a peer connection. The connection checks + /// the signaling state before calling it, so the transport only needs to + /// produce and apply the descriptions. A faulted task rejects the call. + /// + public interface IRtcPeer : IDisposable + { + /// + /// Creates the SDP of an offer. + /// + /// The state of the connection. + /// The SDP of the offer. + Task CreateOfferAsync(RtcSdpRequest request); + + /// + /// Creates the SDP of an answer to the remote offer. + /// + /// The state of the connection. + /// The SDP of the answer. + Task CreateAnswerAsync(RtcSdpRequest request); + + /// + /// Applies the local description, which usually starts gathering + /// candidates. + /// + /// The description to apply. + /// The task representing the operation. + Task SetLocalDescriptionAsync(RtcSessionDescription description); + + /// + /// Applies the remote description. + /// + /// The description to apply. + /// The task representing the operation. + Task SetRemoteDescriptionAsync(RtcSessionDescription description); + + /// + /// Reverts the pending descriptions to the last stable state. + /// + /// The task representing the operation. + Task RollbackAsync(); + + /// + /// Adds a remote candidate, or signals the end of the remote + /// candidates when null. + /// + /// The candidate to add, if any. + /// The task representing the operation. + Task AddIceCandidateAsync(RtcIceCandidate candidate); + + /// + /// Applies a changed configuration. + /// + /// The new configuration. + void SetConfiguration(RtcConfiguration configuration); + + /// + /// Registers a data channel created locally. The transport opens it + /// via once possible. + /// + /// The created channel. + void CreateDataChannel(RtcDataChannel channel); + + /// + /// Sends a message, i.e., a string or a byte array, on an open channel. + /// + /// The channel sending the message. + /// The message. + void Send(RtcDataChannel channel, Object data); + + /// + /// Closes a data channel. The transport finishes the closing via + /// . + /// + /// The channel to close. + void CloseDataChannel(RtcDataChannel channel); + } + + /// + /// Represents the peer connection as seen by its transport. The calls may + /// come from any thread; their events are dispatched as tasks of the + /// connection's event loop. + /// + public interface IRtcPeerHost + { + /// + /// Gets the data channels of the connection. + /// + IReadOnlyList DataChannels { get; } + + /// + /// Surfaces a gathered local candidate, or the end of the candidates + /// when null. + /// + /// The gathered candidate, if any. + void AddLocalCandidate(RtcIceCandidateInit candidate); + + /// + /// Updates the ICE gathering state. + /// + /// The new state. + void SetIceGatheringState(RtcIceGatheringState state); + + /// + /// Updates the ICE connection state, which also determines the + /// connection state. + /// + /// The new state. + void SetIceConnectionState(RtcIceConnectionState state); + + /// + /// Creates the local counterpart of a channel the remote peer + /// created. It is announced via . + /// + /// The label of the channel. + /// The options of the channel, including its id. + /// The new channel. + RtcDataChannel CreateRemoteDataChannel(String label, RtcDataChannelInit options); + + /// + /// Opens a channel created via + /// and raises the datachannel event for it. + /// + /// The channel to announce. + void AnnounceDataChannel(RtcDataChannel channel); + + /// + /// Opens a locally created channel. + /// + /// The channel to open. + /// The id of the channel. + void OpenDataChannel(RtcDataChannel channel, UInt16 id); + + /// + /// Delivers a received message, i.e., a string or a byte array. + /// + /// The channel receiving the message. + /// The message. + void ReceiveMessage(RtcDataChannel channel, Object data); + + /// + /// Finishes closing a channel, which may have been closed by either + /// peer. + /// + /// The closed channel. + void CloseDataChannel(RtcDataChannel channel); + } + + /// + /// Represents the state of a connection when creating a description. + /// + public sealed class RtcSdpRequest + { + internal RtcSdpRequest(Boolean iceRestart, Boolean hasDataChannels, RtcSessionDescription local, RtcSessionDescription remote) + { + IceRestart = iceRestart; + HasDataChannels = hasDataChannels; + LocalDescription = local; + RemoteDescription = remote; + } + + /// + /// Gets if the description should use new ICE credentials. + /// + public Boolean IceRestart { get; } + + /// + /// Gets if the connection has data channels, i.e., needs an + /// application media description. + /// + public Boolean HasDataChannels { get; } + + /// + /// Gets the current local description, if any. + /// + public RtcSessionDescription LocalDescription { get; } + + /// + /// Gets the current remote description, if any. For an answer, this + /// is the offer to answer. + /// + public RtcSessionDescription RemoteDescription { get; } + } +} diff --git a/src/AngleSharp.Io/WebRtc/LoopbackRtcPlatform.cs b/src/AngleSharp.Io/WebRtc/LoopbackRtcPlatform.cs new file mode 100644 index 0000000..39ce34f --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/LoopbackRtcPlatform.cs @@ -0,0 +1,533 @@ +namespace AngleSharp.Io.WebRtc +{ + using AngleSharp.Dom; + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Linq; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + + /// + /// A platform connecting the peer connections it created with each other + /// in-process, e.g., for signaling tests. The SDP is deterministic and + /// only describes data channels; each peer gathers a single host + /// candidate on 127.0.0.1. A peer connects once it applied both + /// descriptions and received a candidate of the remote peer, which it + /// finds via the ICE username fragment of the remote description. + /// + public sealed class LoopbackRtcPlatform : IRtcPlatform + { + private readonly Object _sync = new Object(); + private readonly Dictionary _peers = new Dictionary(StringComparer.Ordinal); + private Int32 _lastId; + + /// + public IRtcPeer CreatePeer(IRtcPeerHost host, RtcConfiguration configuration) + { + if (host is null) + { + throw new ArgumentNullException(nameof(host)); + } + + var id = Interlocked.Increment(ref _lastId); + return new Peer(this, host, configuration ?? new RtcConfiguration(), id); + } + + private sealed class Peer : IRtcPeer + { + private readonly LoopbackRtcPlatform _platform; + private readonly IRtcPeerHost _host; + private readonly Int32 _id; + private readonly List _channels = new List(); + private readonly Dictionary _ids = new Dictionary(); + private readonly Dictionary _links = new Dictionary(); + + private Boolean _relayOnly; + private Int32 _generation; + private Int32 _version; + private String _localUfrag; + private String _gatheredUfrag; + private Boolean _localHasApplication; + private Boolean _stableLocalHasApplication; + private Boolean _isClient; + private String _remoteUfrag; + private String _stableRemoteUfrag; + private String _restartedFor; + private Peer _remotePeer; + private Peer _stableRemotePeer; + private Boolean _remoteCandidate; + private Boolean _connected; + private Boolean _sctp; + private Boolean _closed; + + public Peer(LoopbackRtcPlatform platform, IRtcPeerHost host, RtcConfiguration configuration, Int32 id) + { + _platform = platform; + _host = host; + _id = id; + _relayOnly = configuration.IceTransportPolicy == RtcIceTransportPolicy.Relay; + } + + private UInt16 Port => (UInt16)(50000 + _id % 10000); + + private String Ufrag => String.Format(CultureInfo.InvariantCulture, "lb{0:x4}{1:x2}", _id, _generation); + + private String Password => String.Format(CultureInfo.InvariantCulture, "loopbackpwd{0:x6}{1:x6}", _id, _generation); + + public Task CreateOfferAsync(RtcSdpRequest request) + { + lock (_platform._sync) + { + if (request.IceRestart) + { + _generation++; + } + + // media descriptions cannot be removed once negotiated + var hasApplication = request.HasDataChannels || HasApplication(request.LocalDescription) || HasApplication(request.RemoteDescription); + return Task.FromResult(BuildSdp("actpass", hasApplication)); + } + } + + public Task CreateAnswerAsync(RtcSdpRequest request) + { + lock (_platform._sync) + { + var offer = request.RemoteDescription?.Sdp ?? String.Empty; + var offerUfrag = GetAttribute(offer, "a=ice-ufrag:"); + + // an offer restarting ICE requires new credentials for the + // answer, too + if (_stableRemoteUfrag is not null && offerUfrag != _stableRemoteUfrag && offerUfrag != _restartedFor) + { + _restartedFor = offerUfrag; + _generation++; + } + + return Task.FromResult(BuildSdp("active", HasApplication(request.RemoteDescription))); + } + } + + public Task SetLocalDescriptionAsync(RtcSessionDescription description) + { + var effects = new List(); + + lock (_platform._sync) + { + var sdp = description.Sdp; + var ufrag = GetAttribute(sdp, "a=ice-ufrag:"); + _localHasApplication = HasApplication(description); + _isClient = sdp.IndexOf("a=setup:active", StringComparison.Ordinal) >= 0; + + if (ufrag is not null) + { + _localUfrag = ufrag; + _platform._peers[ufrag] = this; + } + + if (description.Type == RtcSdpType.Answer) + { + MarkStable(); + } + + if (_localHasApplication && ufrag is not null && _gatheredUfrag != ufrag) + { + _gatheredUfrag = ufrag; + Gather(ufrag, effects); + } + + TryConnect(effects); + } + + Run(effects); + return Task.CompletedTask; + } + + public Task SetRemoteDescriptionAsync(RtcSessionDescription description) + { + if (!description.Sdp.StartsWith("v=0", StringComparison.Ordinal)) + { + throw new DomException(DomError.Syntax); + } + + var effects = new List(); + + lock (_platform._sync) + { + var ufrag = GetAttribute(description.Sdp, "a=ice-ufrag:"); + + if (ufrag != _remoteUfrag) + { + _remoteUfrag = ufrag; + _remoteCandidate = false; + } + + _remotePeer = Resolve(); + + if (description.Type == RtcSdpType.Answer) + { + MarkStable(); + } + + TryConnect(effects); + } + + Run(effects); + return Task.CompletedTask; + } + + public Task RollbackAsync() + { + lock (_platform._sync) + { + _localHasApplication = _stableLocalHasApplication; + _remoteUfrag = _stableRemoteUfrag; + _remotePeer = _stableRemotePeer; + } + + return Task.CompletedTask; + } + + public Task AddIceCandidateAsync(RtcIceCandidate candidate) + { + var effects = new List(); + + lock (_platform._sync) + { + if (candidate is not null) + { + _remotePeer ??= Resolve(); + var remote = _remotePeer; + + if (remote is not null && candidate.Address == "127.0.0.1" && candidate.Port == remote.Port && + (candidate.UsernameFragment is null || candidate.UsernameFragment == _remoteUfrag)) + { + _remoteCandidate = true; + TryConnect(effects); + } + } + } + + Run(effects); + return Task.CompletedTask; + } + + public void SetConfiguration(RtcConfiguration configuration) + { + lock (_platform._sync) + { + _relayOnly = configuration?.IceTransportPolicy == RtcIceTransportPolicy.Relay; + } + } + + public void CreateDataChannel(RtcDataChannel channel) + { + var effects = new List(); + + lock (_platform._sync) + { + _channels.Add(channel); + + if (_sctp && _remotePeer is not null) + { + OpenChannel(this, _remotePeer, channel, effects); + } + } + + Run(effects); + } + + public void Send(RtcDataChannel channel, Object data) + { + var effects = new List(); + + lock (_platform._sync) + { + if (_links.TryGetValue(channel, out var link)) + { + var host = link.Peer._host; + effects.Add(() => host.ReceiveMessage(link.Channel, data)); + } + } + + Run(effects); + } + + public void CloseDataChannel(RtcDataChannel channel) + { + var effects = new List(); + + lock (_platform._sync) + { + var host = _host; + effects.Add(() => host.CloseDataChannel(channel)); + + if (_links.TryGetValue(channel, out var link)) + { + var remoteHost = link.Peer._host; + effects.Add(() => remoteHost.CloseDataChannel(link.Channel)); + link.Peer._links.Remove(link.Channel); + _links.Remove(channel); + } + } + + Run(effects); + } + + public void Dispose() + { + var effects = new List(); + + lock (_platform._sync) + { + if (_closed) + { + return; + } + + _closed = true; + + foreach (var key in _platform._peers.Where(m => m.Value == this).Select(m => m.Key).ToArray()) + { + _platform._peers.Remove(key); + } + + var remote = _remotePeer; + + // the remote peer notices the closed connection like a + // lost transport + if (remote is not null && !remote._closed && remote._remotePeer == this && remote._connected) + { + var remoteHost = remote._host; + remote._sctp = false; + effects.Add(() => remoteHost.SetIceConnectionState(RtcIceConnectionState.Disconnected)); + } + + foreach (var link in _links.Values) + { + var linkedHost = link.Peer._host; + effects.Add(() => linkedHost.CloseDataChannel(link.Channel)); + link.Peer._links.Remove(link.Channel); + } + + _links.Clear(); + } + + Run(effects); + } + + private void MarkStable() + { + _stableLocalHasApplication = _localHasApplication; + _stableRemoteUfrag = _remoteUfrag; + _stableRemotePeer = _remotePeer; + } + + private Peer Resolve() => + _remoteUfrag is not null && _platform._peers.TryGetValue(_remoteUfrag, out var peer) && peer != this && !peer._closed ? peer : null; + + private void Gather(String ufrag, List effects) + { + var host = _host; + var relayOnly = _relayOnly; + var candidate = new RtcIceCandidateInit + { + Candidate = String.Format(CultureInfo.InvariantCulture, "candidate:1 1 udp 2122260223 127.0.0.1 {0} typ host generation 0 ufrag {1}", Port, ufrag), + SdpMid = "0", + SdpMLineIndex = 0, + UsernameFragment = ufrag, + }; + + effects.Add(() => host.SetIceGatheringState(RtcIceGatheringState.Gathering)); + + // there is no relay server, hence no candidate for the relay policy + if (!relayOnly) + { + effects.Add(() => host.AddLocalCandidate(candidate)); + } + + effects.Add(() => host.SetIceGatheringState(RtcIceGatheringState.Complete)); + effects.Add(() => host.AddLocalCandidate(null)); + } + + private void TryConnect(List effects) + { + if (_closed) + { + return; + } + + if (!_connected) + { + _remotePeer ??= Resolve(); + + if (!_localHasApplication || _gatheredUfrag is null || _remotePeer is null || !_remoteCandidate) + { + return; + } + + var host = _host; + _connected = true; + effects.Add(() => host.SetIceConnectionState(RtcIceConnectionState.Checking)); + effects.Add(() => host.SetIceConnectionState(RtcIceConnectionState.Connected)); + } + + var remote = _remotePeer; + + // the SCTP association needs both peers connected to each other + if (!_sctp && remote is not null && remote._connected && remote._remotePeer == this && !remote._closed) + { + _sctp = true; + remote._sctp = true; + OpenChannels(this, remote, effects); + OpenChannels(remote, this, effects); + } + } + + private static void OpenChannels(Peer source, Peer target, List effects) + { + foreach (var channel in source._channels.ToArray()) + { + if (!source._links.ContainsKey(channel)) + { + OpenChannel(source, target, channel, effects); + } + } + } + + private static void OpenChannel(Peer source, Peer target, RtcDataChannel channel, List effects) + { + if (channel.ReadyState != RtcDataChannelState.Connecting) + { + return; + } + + var sourceHost = source._host; + var targetHost = target._host; + + if (channel.Negotiated) + { + var id = channel.Id.Value; + var other = target._channels.FirstOrDefault(m => m.Negotiated && m.Id == id && !target._links.ContainsKey(m) && m.ReadyState == RtcDataChannelState.Connecting); + source._ids[channel] = id; + + // the channel waits until the remote peer creates its + // counterpart with the same id + if (other is not null) + { + target._ids[other] = id; + Link(source, channel, target, other); + effects.Add(() => sourceHost.OpenDataChannel(channel, id)); + effects.Add(() => targetHost.OpenDataChannel(other, id)); + } + } + else + { + var id = AllocateId(source, target); + var remote = targetHost.CreateRemoteDataChannel(channel.Label, new RtcDataChannelInit + { + Ordered = channel.Ordered, + MaxPacketLifeTime = channel.MaxPacketLifeTime, + MaxRetransmits = channel.MaxRetransmits, + Protocol = channel.Protocol, + Negotiated = false, + Id = id, + }); + + source._ids[channel] = id; + target._channels.Add(remote); + target._ids[remote] = id; + Link(source, channel, target, remote); + effects.Add(() => targetHost.AnnounceDataChannel(remote)); + effects.Add(() => sourceHost.OpenDataChannel(channel, id)); + } + } + + private static UInt16 AllocateId(Peer source, Peer target) + { + // like the spec, the DTLS client uses even ids and the server + // odd ones, which avoids collisions between the peers + var id = source._isClient ? 0 : 1; + + while (source._ids.ContainsValue((UInt16)id) || target._ids.ContainsValue((UInt16)id)) + { + id += 2; + } + + return (UInt16)id; + } + + private static void Link(Peer source, RtcDataChannel channel, Peer target, RtcDataChannel other) + { + source._links[channel] = (target, other); + target._links[other] = (source, channel); + } + + private String BuildSdp(String setup, Boolean hasApplication) + { + var sessionId = (4000000000000000000L + _id).ToString(CultureInfo.InvariantCulture); + var fingerprint = String.Join(":", Enumerable.Range(0, 32).Select(i => ((_id * 31 + i * 7) & 0xFF).ToString("X2", CultureInfo.InvariantCulture))); + var sdp = new StringBuilder(); + _version++; + + sdp.Append("v=0\r\n"); + sdp.Append("o=- ").Append(sessionId).Append(' ').Append(_version.ToString(CultureInfo.InvariantCulture)).Append(" IN IP4 127.0.0.1\r\n"); + sdp.Append("s=-\r\n"); + sdp.Append("t=0 0\r\n"); + + if (hasApplication) + { + sdp.Append("a=group:BUNDLE 0\r\n"); + } + + sdp.Append("a=ice-options:trickle\r\n"); + sdp.Append("a=msid-semantic: WMS\r\n"); + + if (hasApplication) + { + sdp.Append("m=application 9 UDP/DTLS/SCTP webrtc-datachannel\r\n"); + sdp.Append("c=IN IP4 0.0.0.0\r\n"); + sdp.Append("a=ice-ufrag:").Append(Ufrag).Append("\r\n"); + sdp.Append("a=ice-pwd:").Append(Password).Append("\r\n"); + sdp.Append("a=fingerprint:sha-256 ").Append(fingerprint).Append("\r\n"); + sdp.Append("a=setup:").Append(setup).Append("\r\n"); + sdp.Append("a=mid:0\r\n"); + sdp.Append("a=sctp-port:5000\r\n"); + sdp.Append("a=max-message-size:262144\r\n"); + } + + return sdp.ToString(); + } + + private static Boolean HasApplication(RtcSessionDescription description) => + description?.Sdp.IndexOf("m=application", StringComparison.Ordinal) >= 0; + + private static String GetAttribute(String sdp, String prefix) + { + foreach (var line in sdp.Split('\n')) + { + var value = line.TrimEnd('\r'); + + if (value.StartsWith(prefix, StringComparison.Ordinal)) + { + return value.Substring(prefix.Length); + } + } + + return null; + } + + private static void Run(List effects) + { + // the host is called outside of the lock, as it may dispatch + // events right away, whose handlers may call the platform + foreach (var effect in effects) + { + effect.Invoke(); + } + } + } + } +} diff --git a/src/AngleSharp.Io/WebRtc/RtcBundlePolicy.cs b/src/AngleSharp.Io/WebRtc/RtcBundlePolicy.cs new file mode 100644 index 0000000..e6206b8 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcBundlePolicy.cs @@ -0,0 +1,27 @@ +namespace AngleSharp.Io.WebRtc; + +using AngleSharp.Attributes; + +/// +/// The policies for bundling media. +/// +[DomLiterals] +[DomName("RTCBundlePolicy")] +public enum RtcBundlePolicy +{ + /// + /// One transport per media type. + /// + [DomName("balanced")] + Balanced, + /// + /// One transport per track. + /// + [DomName("max-compat")] + MaxCompat, + /// + /// A single transport for everything. + /// + [DomName("max-bundle")] + MaxBundle, +} diff --git a/src/AngleSharp.Io/WebRtc/RtcConfiguration.cs b/src/AngleSharp.Io/WebRtc/RtcConfiguration.cs new file mode 100644 index 0000000..868184c --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcConfiguration.cs @@ -0,0 +1,138 @@ +namespace AngleSharp.Io.WebRtc +{ + using AngleSharp.Attributes; + using AngleSharp.Dom; + using System; + using System.Linq; + + /// + /// Represents the configuration of a peer connection. + /// + [DomName("RTCConfiguration")] + public sealed class RtcConfiguration + { + /// + /// Gets or sets the STUN and TURN servers the ICE agent may use. + /// + [DomName("iceServers")] + public RtcIceServer[] IceServers { get; set; } = Array.Empty(); + + /// + /// Gets or sets the candidates the ICE agent may use. + /// + [DomName("iceTransportPolicy")] + public RtcIceTransportPolicy IceTransportPolicy { get; set; } = RtcIceTransportPolicy.All; + + /// + /// Gets or sets the policy for bundling media. + /// + [DomName("bundlePolicy")] + public RtcBundlePolicy BundlePolicy { get; set; } = RtcBundlePolicy.Balanced; + + /// + /// Gets or sets the policy for multiplexing RTCP. + /// + [DomName("rtcpMuxPolicy")] + public RtcRtcpMuxPolicy RtcpMuxPolicy { get; set; } = RtcRtcpMuxPolicy.Require; + + /// + /// Gets or sets the number of candidates to gather in advance. + /// + [DomName("iceCandidatePoolSize")] + public Byte IceCandidatePoolSize { get; set; } + + internal RtcConfiguration Clone() => new RtcConfiguration + { + IceServers = (IceServers ?? Array.Empty()).Where(m => m is not null).Select(m => m.Clone()).ToArray(), + IceTransportPolicy = IceTransportPolicy, + BundlePolicy = BundlePolicy, + RtcpMuxPolicy = RtcpMuxPolicy, + IceCandidatePoolSize = IceCandidatePoolSize, + }; + + internal void Validate() + { + foreach (var server in IceServers ?? Array.Empty()) + { + server?.Validate(); + } + } + } + + /// + /// Represents a STUN or TURN server. + /// + [DomName("RTCIceServer")] + public sealed class RtcIceServer + { + /// + /// Gets or sets the URLs of the server, e.g., stun:stun.example.org. + /// + [DomName("urls")] + public String[] Urls { get; set; } = Array.Empty(); + + /// + /// Gets or sets the username for a TURN server. + /// + [DomName("username")] + public String Username { get; set; } + + /// + /// Gets or sets the credential for a TURN server. + /// + [DomName("credential")] + public String Credential { get; set; } + + internal RtcIceServer Clone() => new RtcIceServer + { + Urls = (Urls ?? Array.Empty()).ToArray(), + Username = Username, + Credential = Credential, + }; + + internal void Validate() + { + if (Urls is null || Urls.Length == 0) + { + throw new DomException(DomError.Syntax); + } + + foreach (var url in Urls) + { + var separator = url?.IndexOf(':') ?? -1; + var scheme = separator > 0 ? url.Substring(0, separator).ToLowerInvariant() : String.Empty; + + if ((scheme != "stun" && scheme != "stuns" && scheme != "turn" && scheme != "turns") || separator == url.Length - 1) + { + throw new DomException(DomError.Syntax); + } + + if ((scheme == "turn" || scheme == "turns") && (Username is null || Credential is null)) + { + throw new DomException(DomError.InvalidAccess); + } + } + } + } + + /// + /// Represents the options for creating an offer. + /// + [DomName("RTCOfferOptions")] + public sealed class RtcOfferOptions + { + /// + /// Gets or sets if the offer restarts ICE, i.e., uses new credentials. + /// + [DomName("iceRestart")] + public Boolean IceRestart { get; set; } + } + + /// + /// Represents the options for creating an answer. + /// + [DomName("RTCAnswerOptions")] + public sealed class RtcAnswerOptions + { + } +} diff --git a/src/AngleSharp.Io/WebRtc/RtcDataChannel.cs b/src/AngleSharp.Io/WebRtc/RtcDataChannel.cs new file mode 100644 index 0000000..2036fe9 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcDataChannel.cs @@ -0,0 +1,326 @@ +namespace AngleSharp.Io.WebRtc +{ + using AngleSharp.Attributes; + using AngleSharp.Dom; + using AngleSharp.Dom.Events; + using System; + using System.Collections.Generic; + + /// + /// Represents the options of a data channel. + /// + [DomName("RTCDataChannelInit")] + public sealed class RtcDataChannelInit + { + /// + /// Gets or sets if the messages are delivered in order. + /// + [DomName("ordered")] + public Boolean Ordered { get; set; } = true; + + /// + /// Gets or sets the time in milliseconds a message may be retransmitted. + /// + [DomName("maxPacketLifeTime")] + public UInt16? MaxPacketLifeTime { get; set; } + + /// + /// Gets or sets how often a message may be retransmitted. + /// + [DomName("maxRetransmits")] + public UInt16? MaxRetransmits { get; set; } + + /// + /// Gets or sets the subprotocol of the channel. + /// + [DomName("protocol")] + public String Protocol { get; set; } = String.Empty; + + /// + /// Gets or sets if the channel is negotiated by the application, i.e., + /// created on both peers with the same id instead of announced. + /// + [DomName("negotiated")] + public Boolean Negotiated { get; set; } + + /// + /// Gets or sets the id of the channel, required if negotiated. + /// + [DomName("id")] + public UInt16? Id { get; set; } + } + + /// + /// Represents a data channel of a peer connection. Messages are + /// delivered by the platform; reliability parameters are informational. + /// + [DomName("RTCDataChannel")] + public sealed class RtcDataChannel : EventTarget + { + private readonly RtcPeerConnection _connection; + private readonly Queue _early; + private RtcDataChannelState _readyState; + private String _binaryType; + + internal RtcDataChannel(RtcPeerConnection connection, String label, RtcDataChannelInit options) + { + _connection = connection; + _early = new Queue(); + _readyState = RtcDataChannelState.Connecting; + _binaryType = "arraybuffer"; + Label = label; + Ordered = options.Ordered; + MaxPacketLifeTime = options.MaxPacketLifeTime; + MaxRetransmits = options.MaxRetransmits; + Protocol = options.Protocol ?? String.Empty; + Negotiated = options.Negotiated; + Id = options.Id; + } + + /// + /// Gets the label of the channel. + /// + [DomName("label")] + public String Label { get; } + + /// + /// Gets if the messages are delivered in order. + /// + [DomName("ordered")] + public Boolean Ordered { get; } + + /// + /// Gets the time in milliseconds a message may be retransmitted. + /// + [DomName("maxPacketLifeTime")] + public UInt16? MaxPacketLifeTime { get; } + + /// + /// Gets how often a message may be retransmitted. + /// + [DomName("maxRetransmits")] + public UInt16? MaxRetransmits { get; } + + /// + /// Gets the subprotocol of the channel. + /// + [DomName("protocol")] + public String Protocol { get; } + + /// + /// Gets if the channel is negotiated by the application. + /// + [DomName("negotiated")] + public Boolean Negotiated { get; } + + /// + /// Gets the id of the channel, which is known once it is open unless + /// it was given. + /// + [DomName("id")] + public UInt16? Id { get; private set; } + + /// + /// Gets the state of the channel. + /// + [DomName("readyState")] + public RtcDataChannelState ReadyState => _readyState; + + /// + /// Gets the number of queued bytes, which is always 0 as messages are + /// handed to the platform right away. + /// + [DomName("bufferedAmount")] + public UInt32 BufferedAmount => 0; + + /// + /// Gets or sets the threshold for the bufferedamountlow event. + /// + [DomName("bufferedAmountLowThreshold")] + public UInt32 BufferedAmountLowThreshold { get; set; } + + /// + /// Gets or sets the type of binary messages. Binary messages are + /// always delivered as byte arrays, i.e., array buffers. + /// + [DomName("binaryType")] + public String BinaryType + { + get => _binaryType; + set + { + if (value == "arraybuffer" || value == "blob") + { + _binaryType = value; + } + } + } + + /// + /// Adds or removes the handler for the open event. + /// + [DomName("onopen")] + public event DomEventHandler Opened + { + add { AddEventListener(RtcEventNames.Open, value, false); } + remove { RemoveEventListener(RtcEventNames.Open, value, false); } + } + + /// + /// Adds or removes the handler for the message event. + /// + [DomName("onmessage")] + public event DomEventHandler Message + { + add { AddEventListener(RtcEventNames.Message, value, false); } + remove { RemoveEventListener(RtcEventNames.Message, value, false); } + } + + /// + /// Adds or removes the handler for the closing event. + /// + [DomName("onclosing")] + public event DomEventHandler Closing + { + add { AddEventListener(RtcEventNames.Closing, value, false); } + remove { RemoveEventListener(RtcEventNames.Closing, value, false); } + } + + /// + /// Adds or removes the handler for the close event. + /// + [DomName("onclose")] + public event DomEventHandler Closed + { + add { AddEventListener(RtcEventNames.Close, value, false); } + remove { RemoveEventListener(RtcEventNames.Close, value, false); } + } + + /// + /// Adds or removes the handler for the error event. + /// + [DomName("onerror")] + public event DomEventHandler Error + { + add { AddEventListener(RtcEventNames.Error, value, false); } + remove { RemoveEventListener(RtcEventNames.Error, value, false); } + } + + /// + /// Adds or removes the handler for the bufferedamountlow event. + /// + [DomName("onbufferedamountlow")] + public event DomEventHandler BufferedAmountLow + { + add { AddEventListener(RtcEventNames.BufferedAmountLow, value, false); } + remove { RemoveEventListener(RtcEventNames.BufferedAmountLow, value, false); } + } + + /// + /// Sends a text message. + /// + /// The message to send. + [DomName("send")] + public void Send(String data) => SendCore(data ?? String.Empty); + + /// + /// Sends a binary message. + /// + /// The message to send. + [DomName("send")] + public void Send(Byte[] data) => SendCore((Byte[])(data ?? Array.Empty()).Clone()); + + /// + /// Closes the channel on both peers. + /// + [DomName("close")] + public void Close() + { + if (_readyState == RtcDataChannelState.Closing || _readyState == RtcDataChannelState.Closed) + { + return; + } + + _readyState = RtcDataChannelState.Closing; + _connection.Peer?.CloseDataChannel(this); + } + + internal void Open(UInt16 id) + { + if (_readyState == RtcDataChannelState.Connecting) + { + Id = id; + _readyState = RtcDataChannelState.Open; + Dispatch(new Event(RtcEventNames.Open)); + DeliverEarlyMessages(); + } + } + + internal void Announce() + { + // a remote channel is already open when the datachannel event + // announces it, so it has no open event + if (_readyState == RtcDataChannelState.Connecting) + { + _readyState = RtcDataChannelState.Open; + } + } + + internal void Receive(Object data) + { + // the remote peer may send right after its channel opened, i.e., + // before the open or datachannel event of this channel ran + if (_readyState == RtcDataChannelState.Connecting) + { + _early.Enqueue(data); + } + else if (_readyState == RtcDataChannelState.Open) + { + Dispatch(new MessageEvent(RtcEventNames.Message, false, false, data, String.Empty, String.Empty, null, Array.Empty())); + } + } + + internal void DeliverEarlyMessages() + { + while (_early.Count > 0) + { + Receive(_early.Dequeue()); + } + } + + internal void Finish() + { + if (_readyState == RtcDataChannelState.Closed) + { + return; + } + + // a channel closed by the remote peer or the transport passes the + // closing state with an event, a local close already set it + if (_readyState != RtcDataChannelState.Closing) + { + _readyState = RtcDataChannelState.Closing; + Dispatch(new Event(RtcEventNames.Closing)); + } + + _readyState = RtcDataChannelState.Closed; + Dispatch(new Event(RtcEventNames.Close)); + } + + internal void CloseSilently() + { + _early.Clear(); + _readyState = RtcDataChannelState.Closed; + } + + private void SendCore(Object data) + { + if (_readyState != RtcDataChannelState.Open) + { + throw new DomException(DomError.InvalidState); + } + + _connection.Peer?.Send(this, data); + } + } +} diff --git a/src/AngleSharp.Io/WebRtc/RtcDataChannelState.cs b/src/AngleSharp.Io/WebRtc/RtcDataChannelState.cs new file mode 100644 index 0000000..5abeaa3 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcDataChannelState.cs @@ -0,0 +1,32 @@ +namespace AngleSharp.Io.WebRtc; + +using AngleSharp.Attributes; + +/// +/// The states of a data channel. +/// +[DomLiterals] +[DomName("RTCDataChannelState")] +public enum RtcDataChannelState +{ + /// + /// The channel is not open yet. + /// + [DomName("connecting")] + Connecting, + /// + /// The channel can be used. + /// + [DomName("open")] + Open, + /// + /// The channel is being closed. + /// + [DomName("closing")] + Closing, + /// + /// The channel is closed. + /// + [DomName("closed")] + Closed, +} diff --git a/src/AngleSharp.Io/WebRtc/RtcEvents.cs b/src/AngleSharp.Io/WebRtc/RtcEvents.cs new file mode 100644 index 0000000..b0b2912 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcEvents.cs @@ -0,0 +1,80 @@ +namespace AngleSharp.Io.WebRtc +{ + using AngleSharp.Attributes; + using AngleSharp.Dom.Events; + using System; + + /// + /// Represents the event surfacing a gathered ICE candidate. + /// + [DomName("RTCPeerConnectionIceEvent")] + public sealed class RtcPeerConnectionIceEvent : Event + { + /// + /// Creates a new ICE candidate event. + /// + /// The type of the event. + /// The candidate, or null at the end of the candidates. + /// The URL of the server that found the candidate. + public RtcPeerConnectionIceEvent(String type, RtcIceCandidate candidate, String url = null) + : base(type) + { + Candidate = candidate; + Url = url; + } + + /// + /// Gets the candidate, or null at the end of the candidates. + /// + [DomName("candidate")] + public RtcIceCandidate Candidate { get; } + + /// + /// Gets the URL of the server that found the candidate, if any. + /// + [DomName("url")] + public String Url { get; } + } + + /// + /// Represents the event announcing a data channel opened by the remote + /// peer. + /// + [DomName("RTCDataChannelEvent")] + public sealed class RtcDataChannelEvent : Event + { + /// + /// Creates a new data channel event. + /// + /// The type of the event. + /// The announced channel. + public RtcDataChannelEvent(String type, RtcDataChannel channel) + : base(type) + { + Channel = channel ?? throw new ArgumentNullException(nameof(channel)); + } + + /// + /// Gets the announced channel. + /// + [DomName("channel")] + public RtcDataChannel Channel { get; } + } + + static class RtcEventNames + { + public static readonly String NegotiationNeeded = "negotiationneeded"; + public static readonly String IceCandidate = "icecandidate"; + public static readonly String SignalingStateChange = "signalingstatechange"; + public static readonly String IceGatheringStateChange = "icegatheringstatechange"; + public static readonly String IceConnectionStateChange = "iceconnectionstatechange"; + public static readonly String ConnectionStateChange = "connectionstatechange"; + public static readonly String DataChannel = "datachannel"; + public static readonly String Open = "open"; + public static readonly String Message = "message"; + public static readonly String Closing = "closing"; + public static readonly String Close = "close"; + public static readonly String Error = "error"; + public static readonly String BufferedAmountLow = "bufferedamountlow"; + } +} diff --git a/src/AngleSharp.Io/WebRtc/RtcIceCandidate.cs b/src/AngleSharp.Io/WebRtc/RtcIceCandidate.cs new file mode 100644 index 0000000..612480a --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcIceCandidate.cs @@ -0,0 +1,223 @@ +namespace AngleSharp.Io.WebRtc +{ + using AngleSharp.Attributes; + using System; + using System.Globalization; + + /// + /// Represents the dictionary describing an ICE candidate. + /// + [DomName("RTCIceCandidateInit")] + public sealed class RtcIceCandidateInit + { + /// + /// Gets or sets the candidate-attribute, e.g., "candidate:1 1 udp ...". + /// An empty value signals the end of the candidates. + /// + [DomName("candidate")] + public String Candidate { get; set; } = String.Empty; + + /// + /// Gets or sets the media stream identification of the candidate. + /// + [DomName("sdpMid")] + public String SdpMid { get; set; } + + /// + /// Gets or sets the index of the media description of the candidate. + /// + [DomName("sdpMLineIndex")] + public UInt16? SdpMLineIndex { get; set; } + + /// + /// Gets or sets the ICE username fragment of the candidate. + /// + [DomName("usernameFragment")] + public String UsernameFragment { get; set; } + } + + /// + /// Represents an ICE candidate. The attributes are parsed from the + /// candidate-attribute and stay null if it cannot be parsed. + /// + [DomName("RTCIceCandidate")] + public sealed class RtcIceCandidate + { + /// + /// Creates a new ICE candidate. + /// + /// The description of the candidate. + [DomConstructor] + public RtcIceCandidate(RtcIceCandidateInit candidate) + { + if (candidate is null || (candidate.SdpMid is null && candidate.SdpMLineIndex is null)) + { + throw new InvalidOperationException("Either sdpMid or sdpMLineIndex of the candidate is required."); + } + + Candidate = candidate.Candidate ?? String.Empty; + SdpMid = candidate.SdpMid; + SdpMLineIndex = candidate.SdpMLineIndex; + UsernameFragment = candidate.UsernameFragment; + Parse(); + } + + /// + /// Gets the candidate-attribute. + /// + [DomName("candidate")] + public String Candidate { get; } + + /// + /// Gets the media stream identification of the candidate. + /// + [DomName("sdpMid")] + public String SdpMid { get; } + + /// + /// Gets the index of the media description of the candidate. + /// + [DomName("sdpMLineIndex")] + public UInt16? SdpMLineIndex { get; } + + /// + /// Gets the foundation of the candidate. + /// + [DomName("foundation")] + public String Foundation { get; private set; } + + /// + /// Gets the component of the candidate. + /// + [DomName("component")] + public RtcIceComponent? Component { get; private set; } + + /// + /// Gets the priority of the candidate. + /// + [DomName("priority")] + public UInt32? Priority { get; private set; } + + /// + /// Gets the address of the candidate. + /// + [DomName("address")] + public String Address { get; private set; } + + /// + /// Gets the transport protocol of the candidate. + /// + [DomName("protocol")] + public RtcIceProtocol? Protocol { get; private set; } + + /// + /// Gets the port of the candidate. + /// + [DomName("port")] + public UInt16? Port { get; private set; } + + /// + /// Gets the type of the candidate. + /// + [DomName("type")] + public RtcIceCandidateType? Type { get; private set; } + + /// + /// Gets the TCP type of a TCP candidate. + /// + [DomName("tcpType")] + public RtcIceTcpCandidateType? TcpType { get; private set; } + + /// + /// Gets the related address of a non-host candidate. + /// + [DomName("relatedAddress")] + public String RelatedAddress { get; private set; } + + /// + /// Gets the related port of a non-host candidate. + /// + [DomName("relatedPort")] + public UInt16? RelatedPort { get; private set; } + + /// + /// Gets the ICE username fragment of the candidate. + /// + [DomName("usernameFragment")] + public String UsernameFragment { get; private set; } + + /// + /// Gets the dictionary form of the candidate, e.g., for sending it to + /// the other peer. + /// + /// The description of the candidate. + [DomName("toJSON")] + public RtcIceCandidateInit ToJson() => new RtcIceCandidateInit + { + Candidate = Candidate, + SdpMid = SdpMid, + SdpMLineIndex = SdpMLineIndex, + UsernameFragment = UsernameFragment, + }; + + private void Parse() + { + var value = Candidate.StartsWith("candidate:", StringComparison.Ordinal) ? Candidate.Substring(10) : Candidate; + var parts = value.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + + if (parts.Length < 8 || parts[6] != "typ" || + !TryParseEnum(parts[2].ToLowerInvariant(), out var protocol) || + !UInt32.TryParse(parts[3], NumberStyles.None, CultureInfo.InvariantCulture, out var priority) || + !UInt16.TryParse(parts[5], NumberStyles.None, CultureInfo.InvariantCulture, out var port) || + !TryParseEnum(parts[7], out var type)) + { + return; + } + + Foundation = parts[0]; + Component = parts[1] == "1" ? RtcIceComponent.Rtp : parts[1] == "2" ? RtcIceComponent.Rtcp : (RtcIceComponent?)null; + Protocol = protocol; + Priority = priority; + Address = parts[4]; + Port = port; + Type = type; + + for (var i = 8; i + 1 < parts.Length; i += 2) + { + var name = parts[i]; + var argument = parts[i + 1]; + + if (name == "raddr") + { + RelatedAddress = argument; + } + else if (name == "rport" && UInt16.TryParse(argument, NumberStyles.None, CultureInfo.InvariantCulture, out var relatedPort)) + { + RelatedPort = relatedPort; + } + else if (name == "tcptype" && TryParseEnum(argument, out var tcpType)) + { + TcpType = tcpType; + } + else if (name == "ufrag" && UsernameFragment is null) + { + UsernameFragment = argument; + } + } + } + + private static Boolean TryParseEnum(String value, out T result) + where T : struct + { + // the literals are lowercase words, which the enum names match + // case-insensitively; numbers are not literals + if (value.Length > 0 && Char.IsLetter(value[0]) && Enum.TryParse(value, true, out result)) + { + return true; + } + + result = default; + return false; + } + } +} diff --git a/src/AngleSharp.Io/WebRtc/RtcIceCandidateType.cs b/src/AngleSharp.Io/WebRtc/RtcIceCandidateType.cs new file mode 100644 index 0000000..bc81506 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcIceCandidateType.cs @@ -0,0 +1,32 @@ +namespace AngleSharp.Io.WebRtc; + +using AngleSharp.Attributes; + +/// +/// The types of ICE candidates. +/// +[DomLiterals] +[DomName("RTCIceCandidateType")] +public enum RtcIceCandidateType +{ + /// + /// An address of the host itself. + /// + [DomName("host")] + Host, + /// + /// A server reflexive address, e.g., from STUN. + /// + [DomName("srflx")] + Srflx, + /// + /// A peer reflexive address. + /// + [DomName("prflx")] + Prflx, + /// + /// A relayed address, e.g., from TURN. + /// + [DomName("relay")] + Relay, +} diff --git a/src/AngleSharp.Io/WebRtc/RtcIceComponent.cs b/src/AngleSharp.Io/WebRtc/RtcIceComponent.cs new file mode 100644 index 0000000..0400cf7 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcIceComponent.cs @@ -0,0 +1,22 @@ +namespace AngleSharp.Io.WebRtc; + +using AngleSharp.Attributes; + +/// +/// The components of ICE candidates. +/// +[DomLiterals] +[DomName("RTCIceComponent")] +public enum RtcIceComponent +{ + /// + /// The RTP component. + /// + [DomName("rtp")] + Rtp, + /// + /// The RTCP component. + /// + [DomName("rtcp")] + Rtcp, +} diff --git a/src/AngleSharp.Io/WebRtc/RtcIceConnectionState.cs b/src/AngleSharp.Io/WebRtc/RtcIceConnectionState.cs new file mode 100644 index 0000000..9a58505 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcIceConnectionState.cs @@ -0,0 +1,47 @@ +namespace AngleSharp.Io.WebRtc; + +using AngleSharp.Attributes; + +/// +/// The states of the ICE connection. +/// +[DomLiterals] +[DomName("RTCIceConnectionState")] +public enum RtcIceConnectionState +{ + /// + /// No connectivity checks have started. + /// + [DomName("new")] + New, + /// + /// Connectivity checks are running. + /// + [DomName("checking")] + Checking, + /// + /// A usable connection has been found. + /// + [DomName("connected")] + Connected, + /// + /// The connectivity checks are done. + /// + [DomName("completed")] + Completed, + /// + /// The connection was lost. + /// + [DomName("disconnected")] + Disconnected, + /// + /// No usable connection has been found. + /// + [DomName("failed")] + Failed, + /// + /// The connection is closed. + /// + [DomName("closed")] + Closed, +} diff --git a/src/AngleSharp.Io/WebRtc/RtcIceGatheringState.cs b/src/AngleSharp.Io/WebRtc/RtcIceGatheringState.cs new file mode 100644 index 0000000..c835f7f --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcIceGatheringState.cs @@ -0,0 +1,27 @@ +namespace AngleSharp.Io.WebRtc; + +using AngleSharp.Attributes; + +/// +/// The states of gathering ICE candidates. +/// +[DomLiterals] +[DomName("RTCIceGatheringState")] +public enum RtcIceGatheringState +{ + /// + /// No candidates have been gathered yet. + /// + [DomName("new")] + New, + /// + /// Candidates are being gathered. + /// + [DomName("gathering")] + Gathering, + /// + /// All candidates have been gathered. + /// + [DomName("complete")] + Complete, +} diff --git a/src/AngleSharp.Io/WebRtc/RtcIceProtocol.cs b/src/AngleSharp.Io/WebRtc/RtcIceProtocol.cs new file mode 100644 index 0000000..fdf41f2 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcIceProtocol.cs @@ -0,0 +1,22 @@ +namespace AngleSharp.Io.WebRtc; + +using AngleSharp.Attributes; + +/// +/// The transport protocols of ICE candidates. +/// +[DomLiterals] +[DomName("RTCIceProtocol")] +public enum RtcIceProtocol +{ + /// + /// UDP. + /// + [DomName("udp")] + Udp, + /// + /// TCP. + /// + [DomName("tcp")] + Tcp, +} diff --git a/src/AngleSharp.Io/WebRtc/RtcIceTcpCandidateType.cs b/src/AngleSharp.Io/WebRtc/RtcIceTcpCandidateType.cs new file mode 100644 index 0000000..2dfe79d --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcIceTcpCandidateType.cs @@ -0,0 +1,27 @@ +namespace AngleSharp.Io.WebRtc; + +using AngleSharp.Attributes; + +/// +/// The types of TCP candidates. +/// +[DomLiterals] +[DomName("RTCIceTcpCandidateType")] +public enum RtcIceTcpCandidateType +{ + /// + /// The candidate opens connections. + /// + [DomName("active")] + Active, + /// + /// The candidate accepts connections. + /// + [DomName("passive")] + Passive, + /// + /// The candidate opens connections simultaneously. + /// + [DomName("so")] + So, +} diff --git a/src/AngleSharp.Io/WebRtc/RtcIceTransportPolicy.cs b/src/AngleSharp.Io/WebRtc/RtcIceTransportPolicy.cs new file mode 100644 index 0000000..e2981d1 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcIceTransportPolicy.cs @@ -0,0 +1,22 @@ +namespace AngleSharp.Io.WebRtc; + +using AngleSharp.Attributes; + +/// +/// The candidates the ICE agent may use. +/// +[DomLiterals] +[DomName("RTCIceTransportPolicy")] +public enum RtcIceTransportPolicy +{ + /// + /// All candidates may be used. + /// + [DomName("all")] + All, + /// + /// Only relayed candidates may be used. + /// + [DomName("relay")] + Relay, +} diff --git a/src/AngleSharp.Io/WebRtc/RtcPeerConnection.cs b/src/AngleSharp.Io/WebRtc/RtcPeerConnection.cs new file mode 100644 index 0000000..17f68de --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcPeerConnection.cs @@ -0,0 +1,943 @@ +namespace AngleSharp.Io.WebRtc +{ + using AngleSharp.Attributes; + using AngleSharp.Browser; + using AngleSharp.Dom; + using AngleSharp.Dom.Events; + using System; + using System.Collections.Generic; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Represents the RTCPeerConnection interface, i.e., the signaling state + /// machine of WebRTC. Transport and media are provided by an + /// . + /// + [DomName("RTCPeerConnection")] + [DomExposed("Window")] + public sealed class RtcPeerConnection : EventTarget, IDisposable + { + #region Fields + + private readonly IEventLoop _eventLoop; + private readonly SemaphoreSlim _operations; + private readonly List _channels; + private readonly Host _host; + private readonly IRtcPeer _peer; + + private RtcConfiguration _configuration; + private RtcSessionDescription _currentLocal; + private RtcSessionDescription _pendingLocal; + private RtcSessionDescription _currentRemote; + private RtcSessionDescription _pendingRemote; + private RtcSignalingState _signalingState; + private RtcIceGatheringState _iceGatheringState; + private RtcIceConnectionState _iceConnectionState; + private RtcPeerConnectionState _connectionState; + private Boolean? _canTrickle; + private String _lastCreatedOffer; + private String _lastCreatedAnswer; + private Boolean _lastCreatedOfferRestartsIce; + private Boolean _iceRestartPending; + private Boolean _negotiationNeeded; + private Boolean _closed; + + #endregion + + #region ctor + + /// + /// Creates a new peer connection using the platform registered for the + /// window's browsing context. + /// + /// The window creating the connection. + [DomConstructor] + public RtcPeerConnection(IWindow window) + : this(window, null) + { + } + + /// + /// Creates a new peer connection using the platform registered for the + /// window's browsing context. + /// + /// The window creating the connection. + /// The configuration of the connection. + [DomConstructor] + public RtcPeerConnection(IWindow window, RtcConfiguration configuration) + : this(GetPlatform(window), configuration, window.Document?.Context?.GetService()) + { + } + + /// + /// Creates a new peer connection using the given platform. + /// + /// The platform providing the transport. + /// The configuration of the connection. + /// + /// The event loop for dispatching the events of the transport. Without + /// one, they are dispatched once the running operation completes. + /// + public RtcPeerConnection(IRtcPlatform platform, RtcConfiguration configuration = null, IEventLoop eventLoop = null) + { + if (platform is null) + { + throw new ArgumentNullException(nameof(platform)); + } + + _configuration = (configuration ?? new RtcConfiguration()).Clone(); + _configuration.Validate(); + _eventLoop = eventLoop; + _operations = new SemaphoreSlim(1, 1); + _channels = new List(); + _host = new Host(this); + _peer = platform.CreatePeer(_host, _configuration.Clone()) ?? throw new InvalidOperationException("The platform did not create a peer."); + } + + #endregion + + #region Properties + + /// + /// Gets the pending local description, or the current one if there is + /// no pending one. + /// + [DomName("localDescription")] + public RtcSessionDescription LocalDescription => _pendingLocal ?? _currentLocal; + + /// + /// Gets the local description of the last completed negotiation. + /// + [DomName("currentLocalDescription")] + public RtcSessionDescription CurrentLocalDescription => _currentLocal; + + /// + /// Gets the local description of the running negotiation. + /// + [DomName("pendingLocalDescription")] + public RtcSessionDescription PendingLocalDescription => _pendingLocal; + + /// + /// Gets the pending remote description, or the current one if there + /// is no pending one. + /// + [DomName("remoteDescription")] + public RtcSessionDescription RemoteDescription => _pendingRemote ?? _currentRemote; + + /// + /// Gets the remote description of the last completed negotiation. + /// + [DomName("currentRemoteDescription")] + public RtcSessionDescription CurrentRemoteDescription => _currentRemote; + + /// + /// Gets the remote description of the running negotiation. + /// + [DomName("pendingRemoteDescription")] + public RtcSessionDescription PendingRemoteDescription => _pendingRemote; + + /// + /// Gets the state of the signaling. + /// + [DomName("signalingState")] + public RtcSignalingState SignalingState => _signalingState; + + /// + /// Gets the state of gathering ICE candidates. + /// + [DomName("iceGatheringState")] + public RtcIceGatheringState IceGatheringState => _iceGatheringState; + + /// + /// Gets the state of the ICE connection. + /// + [DomName("iceConnectionState")] + public RtcIceConnectionState IceConnectionState => _iceConnectionState; + + /// + /// Gets the aggregated state of the connection. + /// + [DomName("connectionState")] + public RtcPeerConnectionState ConnectionState => _connectionState; + + /// + /// Gets if the remote peer accepts trickled candidates, or null while + /// there is no remote description. + /// + [DomName("canTrickleIceCandidates")] + public Boolean? CanTrickleIceCandidates => _canTrickle; + + internal IRtcPeer Peer => _closed ? null : _peer; + + #endregion + + #region Events + + /// + /// Adds or removes the handler for the negotiationneeded event. + /// + [DomName("onnegotiationneeded")] + public event DomEventHandler NegotiationNeeded + { + add { AddEventListener(RtcEventNames.NegotiationNeeded, value, false); } + remove { RemoveEventListener(RtcEventNames.NegotiationNeeded, value, false); } + } + + /// + /// Adds or removes the handler for the icecandidate event. + /// + [DomName("onicecandidate")] + public event DomEventHandler IceCandidate + { + add { AddEventListener(RtcEventNames.IceCandidate, value, false); } + remove { RemoveEventListener(RtcEventNames.IceCandidate, value, false); } + } + + /// + /// Adds or removes the handler for the signalingstatechange event. + /// + [DomName("onsignalingstatechange")] + public event DomEventHandler SignalingStateChanged + { + add { AddEventListener(RtcEventNames.SignalingStateChange, value, false); } + remove { RemoveEventListener(RtcEventNames.SignalingStateChange, value, false); } + } + + /// + /// Adds or removes the handler for the icegatheringstatechange event. + /// + [DomName("onicegatheringstatechange")] + public event DomEventHandler IceGatheringStateChanged + { + add { AddEventListener(RtcEventNames.IceGatheringStateChange, value, false); } + remove { RemoveEventListener(RtcEventNames.IceGatheringStateChange, value, false); } + } + + /// + /// Adds or removes the handler for the iceconnectionstatechange event. + /// + [DomName("oniceconnectionstatechange")] + public event DomEventHandler IceConnectionStateChanged + { + add { AddEventListener(RtcEventNames.IceConnectionStateChange, value, false); } + remove { RemoveEventListener(RtcEventNames.IceConnectionStateChange, value, false); } + } + + /// + /// Adds or removes the handler for the connectionstatechange event. + /// + [DomName("onconnectionstatechange")] + public event DomEventHandler ConnectionStateChanged + { + add { AddEventListener(RtcEventNames.ConnectionStateChange, value, false); } + remove { RemoveEventListener(RtcEventNames.ConnectionStateChange, value, false); } + } + + /// + /// Adds or removes the handler for the datachannel event. + /// + [DomName("ondatachannel")] + public event DomEventHandler DataChannel + { + add { AddEventListener(RtcEventNames.DataChannel, value, false); } + remove { RemoveEventListener(RtcEventNames.DataChannel, value, false); } + } + + #endregion + + #region Methods + + /// + /// Creates an offer for starting a negotiation. + /// + /// The offer. + [DomName("createOffer")] + public Task CreateOffer() => CreateOffer(null); + + /// + /// Creates an offer for starting a negotiation. + /// + /// The options of the offer. + /// The offer. + [DomName("createOffer")] + public Task CreateOffer(RtcOfferOptions options) => RunAsync(async () => + { + var sdp = await CreateOfferCoreAsync(options).ConfigureAwait(false); + return new RtcSessionDescriptionInit { Type = RtcSdpType.Offer, Sdp = sdp }; + }); + + /// + /// Creates an answer to the remote offer. + /// + /// The answer. + [DomName("createAnswer")] + public Task CreateAnswer() => CreateAnswer(null); + + /// + /// Creates an answer to the remote offer. + /// + /// The options of the answer. + /// The answer. + [DomName("createAnswer")] + public Task CreateAnswer(RtcAnswerOptions options) => RunAsync(async () => + { + var sdp = await CreateAnswerCoreAsync().ConfigureAwait(false); + return new RtcSessionDescriptionInit { Type = RtcSdpType.Answer, Sdp = sdp }; + }); + + /// + /// Creates and applies an offer or answer, depending on the signaling + /// state. + /// + /// The task representing the operation. + [DomName("setLocalDescription")] + public Task SetLocalDescription() => SetLocalDescription((RtcSessionDescriptionInit)null); + + /// + /// Applies the given local description. + /// + /// The description to apply. + /// The task representing the operation. + public Task SetLocalDescription(RtcSessionDescription description) => SetLocalDescription(description?.ToJson()); + + /// + /// Applies the given local description. Without type, an offer or + /// answer is picked depending on the signaling state; without SDP, the + /// last created offer or answer is used. + /// + /// The description to apply. + /// The task representing the operation. + [DomName("setLocalDescription")] + public Task SetLocalDescription(RtcSessionDescriptionInit description) => RunAsync(async () => + { + var type = description?.Type ?? GetImplicitType(); + var sdp = description?.Sdp; + + if (type == RtcSdpType.Rollback) + { + await RollbackAsync().ConfigureAwait(false); + return true; + } + + if (type == RtcSdpType.Offer) + { + if (String.IsNullOrEmpty(sdp)) + { + sdp = _lastCreatedOffer ?? await CreateOfferCoreAsync(null).ConfigureAwait(false); + } + else if (sdp != _lastCreatedOffer) + { + throw new DomException(DomError.InvalidModification); + } + + EnsureSignalingState(RtcSignalingState.Stable, RtcSignalingState.HaveLocalOffer); + } + else + { + EnsureSignalingState(RtcSignalingState.HaveRemoteOffer, RtcSignalingState.HaveLocalPrAnswer); + + if (String.IsNullOrEmpty(sdp)) + { + sdp = _lastCreatedAnswer ?? await CreateAnswerCoreAsync().ConfigureAwait(false); + } + else if (sdp != _lastCreatedAnswer) + { + throw new DomException(DomError.InvalidModification); + } + } + + var applied = new RtcSessionDescription(type, sdp); + await _peer.SetLocalDescriptionAsync(applied).ConfigureAwait(false); + EnsureOpen(); + + if (type == RtcSdpType.Offer) + { + _pendingLocal = applied; + + if (_lastCreatedOfferRestartsIce) + { + _iceRestartPending = false; + } + + SetSignalingState(RtcSignalingState.HaveLocalOffer); + } + else if (type == RtcSdpType.PrAnswer) + { + _pendingLocal = applied; + SetSignalingState(RtcSignalingState.HaveLocalPrAnswer); + } + else + { + _currentLocal = applied; + _currentRemote = _pendingRemote; + _pendingLocal = null; + _pendingRemote = null; + SetSignalingState(RtcSignalingState.Stable); + } + + return true; + }); + + /// + /// Applies the given remote description. + /// + /// The description to apply. + /// The task representing the operation. + public Task SetRemoteDescription(RtcSessionDescription description) => SetRemoteDescription(description?.ToJson()); + + /// + /// Applies the given remote description. A remote offer while a local + /// offer is pending rolls the local offer back. + /// + /// The description to apply. + /// The task representing the operation. + [DomName("setRemoteDescription")] + public Task SetRemoteDescription(RtcSessionDescriptionInit description) => RunAsync(async () => + { + // the spec rejects a missing type with a TypeError + if (description?.Type is null) + { + throw new InvalidOperationException("The type of the remote description is required."); + } + + var type = description.Type.Value; + + if (type == RtcSdpType.Rollback) + { + await RollbackAsync().ConfigureAwait(false); + return true; + } + + if (String.IsNullOrEmpty(description.Sdp)) + { + throw new DomException(DomError.Syntax); + } + + if (type == RtcSdpType.Offer) + { + if (_signalingState == RtcSignalingState.HaveLocalOffer) + { + await _peer.RollbackAsync().ConfigureAwait(false); + _pendingLocal = null; + _signalingState = RtcSignalingState.Stable; + } + + EnsureSignalingState(RtcSignalingState.Stable, RtcSignalingState.HaveRemoteOffer); + } + else + { + EnsureSignalingState(RtcSignalingState.HaveLocalOffer, RtcSignalingState.HaveRemotePrAnswer); + } + + var applied = new RtcSessionDescription(type, description.Sdp); + await _peer.SetRemoteDescriptionAsync(applied).ConfigureAwait(false); + EnsureOpen(); + _canTrickle = applied.Sdp.IndexOf("a=ice-options:trickle", StringComparison.Ordinal) >= 0; + + if (type == RtcSdpType.Offer) + { + _pendingRemote = applied; + _lastCreatedAnswer = null; + SetSignalingState(RtcSignalingState.HaveRemoteOffer); + } + else if (type == RtcSdpType.PrAnswer) + { + _pendingRemote = applied; + SetSignalingState(RtcSignalingState.HaveRemotePrAnswer); + } + else + { + _currentRemote = applied; + _currentLocal = _pendingLocal; + _pendingLocal = null; + _pendingRemote = null; + SetSignalingState(RtcSignalingState.Stable); + } + + return true; + }); + + /// + /// Signals the end of the remote candidates. + /// + /// The task representing the operation. + [DomName("addIceCandidate")] + public Task AddIceCandidate() => AddIceCandidate((RtcIceCandidateInit)null); + + /// + /// Adds a remote candidate. + /// + /// The candidate to add. + /// The task representing the operation. + public Task AddIceCandidate(RtcIceCandidate candidate) => AddIceCandidate(candidate?.ToJson()); + + /// + /// Adds a remote candidate. A null candidate or an empty + /// candidate-attribute signals the end of the remote candidates. + /// + /// The candidate to add. + /// The task representing the operation. + [DomName("addIceCandidate")] + public Task AddIceCandidate(RtcIceCandidateInit candidate) => RunAsync(async () => + { + var end = candidate is null || String.IsNullOrEmpty(candidate.Candidate); + + // the spec rejects this with a TypeError + if (!end && candidate.SdpMid is null && candidate.SdpMLineIndex is null) + { + throw new InvalidOperationException("Either sdpMid or sdpMLineIndex of the candidate is required."); + } + + if (RemoteDescription is null) + { + throw new DomException(DomError.InvalidState); + } + + await _peer.AddIceCandidateAsync(end ? null : new RtcIceCandidate(candidate)).ConfigureAwait(false); + return true; + }); + + /// + /// Requests new ICE credentials with the next offer. + /// + [DomName("restartIce")] + public void RestartIce() + { + if (!_closed) + { + _iceRestartPending = true; + UpdateNegotiationNeeded(false); + } + } + + /// + /// Gets the configuration of the connection. + /// + /// A copy of the configuration. + [DomName("getConfiguration")] + public RtcConfiguration GetConfiguration() => _configuration.Clone(); + + /// + /// Changes the configuration of the connection. The bundle and RTCP + /// multiplexing policies cannot be changed, the candidate pool size + /// only before a local description has been applied. + /// + /// The new configuration. + [DomName("setConfiguration")] + public void SetConfiguration(RtcConfiguration configuration) + { + EnsureOpen(); + var updated = (configuration ?? new RtcConfiguration()).Clone(); + + if (updated.BundlePolicy != _configuration.BundlePolicy || + updated.RtcpMuxPolicy != _configuration.RtcpMuxPolicy || + (updated.IceCandidatePoolSize != _configuration.IceCandidatePoolSize && LocalDescription is not null)) + { + throw new DomException(DomError.InvalidModification); + } + + updated.Validate(); + _configuration = updated; + _peer.SetConfiguration(updated.Clone()); + } + + /// + /// Creates a data channel, which opens once the connection is + /// established. + /// + /// The label of the channel. + /// The new channel. + [DomName("createDataChannel")] + public RtcDataChannel CreateDataChannel(String label) => CreateDataChannel(label, null); + + /// + /// Creates a data channel, which opens once the connection is + /// established. + /// + /// The label of the channel. + /// The options of the channel. + /// The new channel. + [DomName("createDataChannel")] + public RtcDataChannel CreateDataChannel(String label, RtcDataChannelInit options) + { + EnsureOpen(); + label = label ?? String.Empty; + options = options ?? new RtcDataChannelInit(); + + // the spec throws a TypeError for these + if (Encoding.UTF8.GetByteCount(label) > 65535 || Encoding.UTF8.GetByteCount(options.Protocol ?? String.Empty) > 65535) + { + throw new InvalidOperationException("The label and protocol of a data channel must not exceed 65535 bytes."); + } + + if (options.MaxPacketLifeTime is not null && options.MaxRetransmits is not null) + { + throw new InvalidOperationException("Only one of maxPacketLifeTime and maxRetransmits may be set."); + } + + if (options.Negotiated && options.Id is null) + { + throw new InvalidOperationException("A negotiated data channel requires an id."); + } + + if (options.Id == 65535) + { + throw new InvalidOperationException("The id of a data channel must be below 65535."); + } + + var channel = new RtcDataChannel(this, label, new RtcDataChannelInit + { + Ordered = options.Ordered, + MaxPacketLifeTime = options.MaxPacketLifeTime, + MaxRetransmits = options.MaxRetransmits, + Protocol = options.Protocol, + Negotiated = options.Negotiated, + Id = options.Negotiated ? options.Id : null, + }); + + lock (_channels) + { + _channels.Add(channel); + } + + _peer.CreateDataChannel(channel); + UpdateNegotiationNeeded(false); + return channel; + } + + /// + /// Closes the connection and its data channels without raising any + /// events. + /// + [DomName("close")] + public void Close() + { + if (_closed) + { + return; + } + + _closed = true; + _signalingState = RtcSignalingState.Closed; + _iceConnectionState = RtcIceConnectionState.Closed; + _connectionState = RtcPeerConnectionState.Closed; + + foreach (var channel in GetChannels()) + { + channel.CloseSilently(); + } + + _peer.Dispose(); + } + + void IDisposable.Dispose() => Close(); + + #endregion + + #region Helpers + + private static IRtcPlatform GetPlatform(IWindow window) + { + if (window is null) + { + throw new ArgumentNullException(nameof(window)); + } + + return window.Document?.Context?.GetService() ?? throw new DomException(DomError.NotSupported); + } + + private async Task RunAsync(Func> operation) + { + // like the spec's operations chain, one operation runs at a time + EnsureOpen(); + await _operations.WaitAsync().ConfigureAwait(false); + _host.BeginOperation(); + + try + { + EnsureOpen(); + return await operation.Invoke().ConfigureAwait(false); + } + finally + { + _operations.Release(); + _host.EndOperation(); + } + } + + private async Task CreateOfferCoreAsync(RtcOfferOptions options) + { + EnsureSignalingState(RtcSignalingState.Stable, RtcSignalingState.HaveLocalOffer); + var restart = _iceRestartPending || (options?.IceRestart ?? false); + var request = new RtcSdpRequest(restart, GetChannels().Length > 0, LocalDescription, RemoteDescription); + var sdp = await _peer.CreateOfferAsync(request).ConfigureAwait(false); + EnsureOpen(); + _lastCreatedOffer = sdp; + _lastCreatedOfferRestartsIce = restart; + return sdp; + } + + private async Task CreateAnswerCoreAsync() + { + EnsureSignalingState(RtcSignalingState.HaveRemoteOffer, RtcSignalingState.HaveLocalPrAnswer); + var request = new RtcSdpRequest(false, GetChannels().Length > 0, LocalDescription, RemoteDescription); + var sdp = await _peer.CreateAnswerAsync(request).ConfigureAwait(false); + EnsureOpen(); + _lastCreatedAnswer = sdp; + return sdp; + } + + private async Task RollbackAsync() + { + EnsureSignalingState(RtcSignalingState.HaveLocalOffer, RtcSignalingState.HaveRemoteOffer); + await _peer.RollbackAsync().ConfigureAwait(false); + EnsureOpen(); + _pendingLocal = null; + _pendingRemote = null; + SetSignalingState(RtcSignalingState.Stable); + } + + private RtcSdpType GetImplicitType() => + _signalingState == RtcSignalingState.Stable || + _signalingState == RtcSignalingState.HaveLocalOffer || + _signalingState == RtcSignalingState.HaveRemotePrAnswer ? RtcSdpType.Offer : RtcSdpType.Answer; + + private void EnsureOpen() + { + if (_closed) + { + throw new DomException(DomError.InvalidState); + } + } + + private void EnsureSignalingState(RtcSignalingState first, RtcSignalingState second) + { + EnsureOpen(); + + if (_signalingState != first && _signalingState != second) + { + throw new DomException(DomError.InvalidState); + } + } + + private void SetSignalingState(RtcSignalingState state) + { + var changed = _signalingState != state; + _signalingState = state; + + if (changed) + { + Dispatch(new Event(RtcEventNames.SignalingStateChange)); + } + + if (state == RtcSignalingState.Stable) + { + UpdateNegotiationNeeded(true); + } + } + + private void UpdateNegotiationNeeded(Boolean negotiated) + { + if (_closed || _signalingState != RtcSignalingState.Stable) + { + return; + } + + var hasApplication = _currentLocal?.Sdp.IndexOf("m=application", StringComparison.Ordinal) >= 0; + var needed = _iceRestartPending || (GetChannels().Length > 0 && !hasApplication); + + if (!needed) + { + _negotiationNeeded = false; + } + else if (!_negotiationNeeded || negotiated) + { + // a negotiation that did not satisfy the need raises the + // event again, like the spec's check after reaching stable + _negotiationNeeded = true; + _host.Post(() => + { + if (!_closed && _negotiationNeeded && _signalingState == RtcSignalingState.Stable) + { + Dispatch(new Event(RtcEventNames.NegotiationNeeded)); + } + }); + } + } + + private RtcDataChannel[] GetChannels() + { + lock (_channels) + { + return _channels.ToArray(); + } + } + + private void OnLocalCandidate(RtcIceCandidateInit candidate) + { + var value = candidate is null ? null : new RtcIceCandidate(candidate); + Dispatch(new RtcPeerConnectionIceEvent(RtcEventNames.IceCandidate, value)); + } + + private void OnIceGatheringState(RtcIceGatheringState state) + { + if (_iceGatheringState != state) + { + _iceGatheringState = state; + Dispatch(new Event(RtcEventNames.IceGatheringStateChange)); + } + } + + private void OnIceConnectionState(RtcIceConnectionState state) + { + if (_iceConnectionState == state) + { + return; + } + + _iceConnectionState = state; + Dispatch(new Event(RtcEventNames.IceConnectionStateChange)); + + // without DTLS transports of its own, the connection state follows + // the ICE connection state + var connectionState = state switch + { + RtcIceConnectionState.Checking => RtcPeerConnectionState.Connecting, + RtcIceConnectionState.Connected => RtcPeerConnectionState.Connected, + RtcIceConnectionState.Completed => RtcPeerConnectionState.Connected, + RtcIceConnectionState.Disconnected => RtcPeerConnectionState.Disconnected, + RtcIceConnectionState.Failed => RtcPeerConnectionState.Failed, + RtcIceConnectionState.Closed => RtcPeerConnectionState.Closed, + _ => RtcPeerConnectionState.New, + }; + + if (_connectionState != connectionState) + { + _connectionState = connectionState; + Dispatch(new Event(RtcEventNames.ConnectionStateChange)); + } + } + + #endregion + + #region Host + + /// + /// Turns the calls of the transport into tasks of the event loop, or, + /// without one, defers them until the running operation completed, + /// so the state of the operation is applied before. + /// + private sealed class Host : IRtcPeerHost + { + private readonly RtcPeerConnection _connection; + private readonly List _deferred; + private Int32 _operations; + + public Host(RtcPeerConnection connection) + { + _connection = connection; + _deferred = new List(); + } + + public IReadOnlyList DataChannels => _connection.GetChannels(); + + public void BeginOperation() + { + lock (_deferred) + { + _operations++; + } + } + + public void EndOperation() + { + Action[] actions; + + lock (_deferred) + { + if (--_operations > 0) + { + return; + } + + actions = _deferred.ToArray(); + _deferred.Clear(); + } + + foreach (var action in actions) + { + action.Invoke(); + } + } + + public void Post(Action action) + { + void Run() + { + if (!_connection._closed) + { + action.Invoke(); + } + } + + if (_connection._eventLoop is IEventLoop eventLoop) + { + eventLoop.Enqueue(_ => Run(), TaskPriority.Normal); + return; + } + + lock (_deferred) + { + if (_operations > 0) + { + _deferred.Add(Run); + return; + } + } + + Run(); + } + + public void AddLocalCandidate(RtcIceCandidateInit candidate) => + Post(() => _connection.OnLocalCandidate(candidate)); + + public void SetIceGatheringState(RtcIceGatheringState state) => + Post(() => _connection.OnIceGatheringState(state)); + + public void SetIceConnectionState(RtcIceConnectionState state) => + Post(() => _connection.OnIceConnectionState(state)); + + public RtcDataChannel CreateRemoteDataChannel(String label, RtcDataChannelInit options) + { + var channel = new RtcDataChannel(_connection, label ?? String.Empty, options ?? new RtcDataChannelInit()); + + lock (_connection._channels) + { + _connection._channels.Add(channel); + } + + return channel; + } + + public void AnnounceDataChannel(RtcDataChannel channel) => Post(() => + { + channel.Announce(); + _connection.Dispatch(new RtcDataChannelEvent(RtcEventNames.DataChannel, channel)); + channel.DeliverEarlyMessages(); + }); + + public void OpenDataChannel(RtcDataChannel channel, UInt16 id) => + Post(() => channel.Open(id)); + + public void ReceiveMessage(RtcDataChannel channel, Object data) => + Post(() => channel.Receive(data)); + + public void CloseDataChannel(RtcDataChannel channel) => + Post(() => channel.Finish()); + } + + #endregion + } +} diff --git a/src/AngleSharp.Io/WebRtc/RtcPeerConnectionState.cs b/src/AngleSharp.Io/WebRtc/RtcPeerConnectionState.cs new file mode 100644 index 0000000..72fc9f1 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcPeerConnectionState.cs @@ -0,0 +1,42 @@ +namespace AngleSharp.Io.WebRtc; + +using AngleSharp.Attributes; + +/// +/// The aggregated states of a peer connection. +/// +[DomLiterals] +[DomName("RTCPeerConnectionState")] +public enum RtcPeerConnectionState +{ + /// + /// No transport is in use yet. + /// + [DomName("new")] + New, + /// + /// A transport is being established. + /// + [DomName("connecting")] + Connecting, + /// + /// The transports are established. + /// + [DomName("connected")] + Connected, + /// + /// A transport was lost. + /// + [DomName("disconnected")] + Disconnected, + /// + /// A transport failed. + /// + [DomName("failed")] + Failed, + /// + /// The connection is closed. + /// + [DomName("closed")] + Closed, +} diff --git a/src/AngleSharp.Io/WebRtc/RtcRtcpMuxPolicy.cs b/src/AngleSharp.Io/WebRtc/RtcRtcpMuxPolicy.cs new file mode 100644 index 0000000..25c3716 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcRtcpMuxPolicy.cs @@ -0,0 +1,17 @@ +namespace AngleSharp.Io.WebRtc; + +using AngleSharp.Attributes; + +/// +/// The policies for multiplexing RTCP. +/// +[DomLiterals] +[DomName("RTCRtcpMuxPolicy")] +public enum RtcRtcpMuxPolicy +{ + /// + /// RTP and RTCP share a transport. + /// + [DomName("require")] + Require, +} diff --git a/src/AngleSharp.Io/WebRtc/RtcSdpType.cs b/src/AngleSharp.Io/WebRtc/RtcSdpType.cs new file mode 100644 index 0000000..b9c1f9f --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcSdpType.cs @@ -0,0 +1,32 @@ +namespace AngleSharp.Io.WebRtc; + +using AngleSharp.Attributes; + +/// +/// The types of session descriptions. +/// +[DomLiterals] +[DomName("RTCSdpType")] +public enum RtcSdpType +{ + /// + /// An offer starting a negotiation. + /// + [DomName("offer")] + Offer, + /// + /// A provisional answer. + /// + [DomName("pranswer")] + PrAnswer, + /// + /// A final answer completing a negotiation. + /// + [DomName("answer")] + Answer, + /// + /// Cancels the current negotiation. + /// + [DomName("rollback")] + Rollback, +} diff --git a/src/AngleSharp.Io/WebRtc/RtcSessionDescription.cs b/src/AngleSharp.Io/WebRtc/RtcSessionDescription.cs new file mode 100644 index 0000000..dc179c9 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcSessionDescription.cs @@ -0,0 +1,78 @@ +namespace AngleSharp.Io.WebRtc +{ + using AngleSharp.Attributes; + using System; + + /// + /// Represents the dictionary describing a session description. + /// + [DomName("RTCSessionDescriptionInit")] + public sealed class RtcSessionDescriptionInit + { + /// + /// Gets or sets the type of the description. May be omitted when + /// setting the local description, which then picks the type. + /// + [DomName("type")] + public RtcSdpType? Type { get; set; } + + /// + /// Gets or sets the SDP of the description. + /// + [DomName("sdp")] + public String Sdp { get; set; } = String.Empty; + } + + /// + /// Represents a session description, i.e., an SDP with its type. + /// + [DomName("RTCSessionDescription")] + public sealed class RtcSessionDescription + { + /// + /// Creates a new session description. + /// + /// The type and SDP of the description. + [DomConstructor] + public RtcSessionDescription(RtcSessionDescriptionInit description) + { + if (description?.Type is null) + { + throw new InvalidOperationException("The type of the session description is required."); + } + + Type = description.Type.Value; + Sdp = description.Sdp ?? String.Empty; + } + + internal RtcSessionDescription(RtcSdpType type, String sdp) + { + Type = type; + Sdp = sdp ?? String.Empty; + } + + /// + /// Gets the type of the description. + /// + [DomName("type")] + public RtcSdpType Type { get; } + + /// + /// Gets the SDP of the description. + /// + [DomName("sdp")] + public String Sdp { get; } + + /// + /// Gets the dictionary form of the description, e.g., for sending it + /// to the other peer. + /// + /// The type and SDP of the description. + [DomName("toJSON")] + public RtcSessionDescriptionInit ToJson() => new RtcSessionDescriptionInit + { + Type = Type, + Sdp = Sdp, + }; + } +} diff --git a/src/AngleSharp.Io/WebRtc/RtcSignalingState.cs b/src/AngleSharp.Io/WebRtc/RtcSignalingState.cs new file mode 100644 index 0000000..f3af2c3 --- /dev/null +++ b/src/AngleSharp.Io/WebRtc/RtcSignalingState.cs @@ -0,0 +1,42 @@ +namespace AngleSharp.Io.WebRtc; + +using AngleSharp.Attributes; + +/// +/// The states of the signaling. +/// +[DomLiterals] +[DomName("RTCSignalingState")] +public enum RtcSignalingState +{ + /// + /// No negotiation is in progress. + /// + [DomName("stable")] + Stable, + /// + /// A local offer has been applied. + /// + [DomName("have-local-offer")] + HaveLocalOffer, + /// + /// A remote offer has been applied. + /// + [DomName("have-remote-offer")] + HaveRemoteOffer, + /// + /// A local provisional answer has been applied. + /// + [DomName("have-local-pranswer")] + HaveLocalPrAnswer, + /// + /// A remote provisional answer has been applied. + /// + [DomName("have-remote-pranswer")] + HaveRemotePrAnswer, + /// + /// The connection is closed. + /// + [DomName("closed")] + Closed, +} From 392eb08337408dc53344cd6dabd6a3028de30204 Mon Sep 17 00:00:00 2001 From: Florian Rappl Date: Fri, 23 Oct 2026 09:09:26 +0200 Subject: [PATCH 09/10] Implemented support for languages #49 --- CHANGELOG.md | 3 +- .../IoConfigurationExtensionsTests.cs | 51 +++++++++++++++++++ src/AngleSharp.Io/Dom/Navigator.cs | 42 +++++++++++++++ src/AngleSharp.Io/Dom/NavigatorExtensions.cs | 20 ++++++++ src/AngleSharp.Io/NavigatorOptions.cs | 9 ++++ 5 files changed, 124 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dfc93a..cd21caa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # 1.2.0 -Released on Wednesday, October 21 2026. +Released on Friday, October 23 2026. - Added an HTTP cache for the requesters (`WithHttpCache`) - Added `IoOptions.CreateTemporary` and `IoOptions.CreatePersistent` @@ -10,6 +10,7 @@ Released on Wednesday, October 21 2026. - Added reporting with network error capture and `ReportingObserver` (#46) - Added `RTCPeerConnection` with data channels and a loopback platform (#40) - Added `navigator.permissions` with a policy gating clipboard, geolocation, and `persist` (#47) +- Added `navigator.language` and `navigator.languages` (`NavigatorOptions.Languages`) (#49) # 1.1.0 diff --git a/src/AngleSharp.Io.Tests/IoConfigurationExtensionsTests.cs b/src/AngleSharp.Io.Tests/IoConfigurationExtensionsTests.cs index 873df14..29a91f6 100644 --- a/src/AngleSharp.Io.Tests/IoConfigurationExtensionsTests.cs +++ b/src/AngleSharp.Io.Tests/IoConfigurationExtensionsTests.cs @@ -11,6 +11,7 @@ namespace AngleSharp.Io.Tests using AngleSharp.Io.StorageManagerApi; using NUnit.Framework; using System; + using System.Globalization; using System.Linq; using System.Net.Http; using System.Threading.Tasks; @@ -95,6 +96,56 @@ public async Task WithIoLeavesOptionalNavigatorApisUnavailableByDefault() Assert.IsNull(navigator.ServiceWorker()); } + [Test] + public async Task WithIoExposesConfiguredNavigatorLanguages() + { + var options = new IoOptions + { + Navigator = new NavigatorOptions { Languages = new[] { "de-AT", "de", "", "DE-at" } }, + }; + + var context = await OpenContextAsync(Configuration.Default.WithIo(options), "https://io.example/").ConfigureAwait(false); + var navigator = GetNavigator(context); + + CollectionAssert.AreEqual(new[] { "de-AT", "de" }, navigator.Languages()); + Assert.AreEqual("de-AT", navigator.Language()); + } + + [Test] + public async Task WithIoDerivesNavigatorLanguagesFromUiCultureByDefault() + { + var previous = CultureInfo.CurrentUICulture; + CultureInfo.CurrentUICulture = new CultureInfo("fr-CA"); + + try + { + var context = await OpenContextAsync(Configuration.Default.WithIo(new IoOptions()), "https://io.example/").ConfigureAwait(false); + var navigator = GetNavigator(context); + + CollectionAssert.AreEqual(new[] { "fr-CA", "fr" }, navigator.Languages()); + Assert.AreEqual("fr-CA", navigator.Language()); + } + finally + { + CultureInfo.CurrentUICulture = previous; + } + } + + [Test] + public async Task NavigatorLanguagesReturnsACopy() + { + var options = new IoOptions + { + Navigator = new NavigatorOptions { Languages = new[] { "en-US", "en" } }, + }; + + var context = await OpenContextAsync(Configuration.Default.WithIo(options), "https://io.example/").ConfigureAwait(false); + var navigator = GetNavigator(context); + navigator.Languages()[0] = "xx"; + + Assert.AreEqual("en-US", navigator.Language()); + } + private static async Task OpenContextAsync(IConfiguration configuration, String address) { var context = BrowsingContext.New(configuration); diff --git a/src/AngleSharp.Io/Dom/Navigator.cs b/src/AngleSharp.Io/Dom/Navigator.cs index 27d899e..c647d06 100644 --- a/src/AngleSharp.Io/Dom/Navigator.cs +++ b/src/AngleSharp.Io/Dom/Navigator.cs @@ -2,21 +2,28 @@ namespace AngleSharp.Io.Dom; using AngleSharp.Browser.Dom; using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; using System.Net.NetworkInformation; sealed class Navigator : INavigator { private readonly IBrowsingContext _context; private readonly String _userAgent; + private readonly String[] _languages; public Navigator(IBrowsingContext context, NavigatorOptions options) { _context = context; _userAgent = options.UserAgent; + _languages = GetLanguages(options.Languages); } public IBrowsingContext Context => _context; + public String[] Languages => _languages; + public String Name => "Netscape"; public String Platform => String.Empty; @@ -50,4 +57,39 @@ public void WaitForStorageUpdates() } public Boolean IsOnline => NetworkInterface.GetIsNetworkAvailable(); + + internal static String[] GetLanguages(IEnumerable languages) + { + var result = (languages ?? Enumerable.Empty()) + .Where(language => !String.IsNullOrWhiteSpace(language)) + .Select(language => language.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + return result.Length > 0 ? result : GetCultureLanguages(CultureInfo.CurrentUICulture); + } + + private static String[] GetCultureLanguages(CultureInfo culture) + { + var languages = new List(); + + // walk up the parent chain (de-AT -> de) like browsers list a regional + // tag followed by its base language; the invariant culture has no name + while (culture != null && !String.IsNullOrEmpty(culture.Name)) + { + if (!languages.Contains(culture.Name)) + { + languages.Add(culture.Name); + } + + culture = culture.Parent; + } + + if (languages.Count == 0) + { + languages.Add("en-US"); + } + + return languages.ToArray(); + } } diff --git a/src/AngleSharp.Io/Dom/NavigatorExtensions.cs b/src/AngleSharp.Io/Dom/NavigatorExtensions.cs index b75015a..7ca67bf 100644 --- a/src/AngleSharp.Io/Dom/NavigatorExtensions.cs +++ b/src/AngleSharp.Io/Dom/NavigatorExtensions.cs @@ -33,6 +33,26 @@ public static Permissions Permissions(this INavigator navigator) return PermissionsByNavigator.GetValue(navigator, _ => new Permissions(policy, () => LockManager.GetOrigin(context.Active?.DefaultView))); } + /// + /// Gets the user's preferred language, i.e., the first entry of + /// navigator.languages. + /// + [DomName("language")] + [DomAccessor(Accessors.Getter)] + public static String Language(this INavigator navigator) => navigator.Languages()[0]; + + /// + /// Gets the user's preferred languages, most preferred first. + /// + [DomName("languages")] + [DomAccessor(Accessors.Getter)] + public static String[] Languages(this INavigator navigator) + { + // navigators not registered by this library fall back to the UI culture + var languages = (navigator as Navigator)?.Languages ?? Navigator.GetLanguages(null); + return (String[])languages.Clone(); + } + /// /// Gets the clipboard object. /// diff --git a/src/AngleSharp.Io/NavigatorOptions.cs b/src/AngleSharp.Io/NavigatorOptions.cs index 93da7cb..2336d93 100644 --- a/src/AngleSharp.Io/NavigatorOptions.cs +++ b/src/AngleSharp.Io/NavigatorOptions.cs @@ -1,6 +1,7 @@ namespace AngleSharp.Io; using System; +using System.Collections.Generic; /// /// Options for setting up the navigator. @@ -11,4 +12,12 @@ public sealed class NavigatorOptions /// Gets or sets the user agent string to use for the navigator. /// public String UserAgent { get; set; } = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36 OPR/32.0.1948.19 (Edition beta)"; + + /// + /// Gets or sets the preferred languages exposed via navigator.languages, + /// most preferred first, as BCP 47 language tags (e.g., "en-US"). The + /// first entry is also exposed via navigator.language. If not set or + /// empty, the languages are derived from the current UI culture. + /// + public IEnumerable Languages { get; set; } } From cc3921432798fcf95f3e374116214b9481dfc2b1 Mon Sep 17 00:00:00 2001 From: Florian Rappl Date: Fri, 23 Oct 2026 09:17:02 +0200 Subject: [PATCH 10/10] .NET Framework 4.6.2 --- src/AngleSharp.Io/AngleSharp.Io.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/AngleSharp.Io/AngleSharp.Io.csproj b/src/AngleSharp.Io/AngleSharp.Io.csproj index 3a93671..e912017 100644 --- a/src/AngleSharp.Io/AngleSharp.Io.csproj +++ b/src/AngleSharp.Io/AngleSharp.Io.csproj @@ -47,6 +47,7 @@ +