From e93edf695d9446b96218d9c644739ce0eef300bb Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 11 Aug 2026 17:10:37 -0400 Subject: [PATCH 01/14] Improve ObjectStore ergonomics for end-of-run persistence - Key rules (charset/extension) move to static ObjectStore.IsSupportedKey; unsupported-key errors now state the rules and point to the new ObjectStore.SanitizeKey helper for programmatically-built keys - Reading a missing key lists the available keys in the error - New Python-facing helpers: save_json/read_json with a tolerant encoder (datetime/date/time, Decimal, numpy scalars, Symbol, non-string dict keys) and read_json default for missing keys, save_dataframe (CSV, or JSON for .json keys) and save_text as an alias of save - Unit tests plus ObjectStoreErgonomicsRegressionAlgorithm (C# + Python) --- ...bjectStoreErgonomicsRegressionAlgorithm.cs | 162 ++++++++++++++ ...bjectStoreErgonomicsRegressionAlgorithm.py | 104 +++++++++ Common/Storage/ObjectStore.cs | 201 ++++++++++++++++++ Engine/Storage/LocalObjectStore.cs | 23 +- Tests/Common/Storage/LocalObjectStoreTests.cs | 156 ++++++++++++++ 5 files changed, 636 insertions(+), 10 deletions(-) create mode 100644 Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs create mode 100644 Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py diff --git a/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs b/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs new file mode 100644 index 000000000000..05a86aed9a37 --- /dev/null +++ b/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs @@ -0,0 +1,162 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using QuantConnect.Interfaces; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Regression algorithm asserting the ObjectStore ergonomics: unsupported keys are rejected with the + /// key rules stated in the error, converts arbitrary names + /// into supported keys, is an alias of Save and reading a + /// missing key lists the available keys in the error. The Python version of this algorithm additionally + /// covers the tolerant save_json/read_json and save_dataframe helpers + /// + public class ObjectStoreErgonomicsRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + /// + /// Initialize the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. + /// + public override void Initialize() + { + SetStartDate(2013, 10, 7); + SetEndDate(2013, 10, 11); + + SetBenchmark(x => 0); + + AddEquity("SPY", Resolution.Daily); + } + + /// + /// End of algorithm run event handler. This method is called at the end of a backtest or live trading operation. + /// + public override void OnEndOfAlgorithm() + { + // an unsupported key, e.g. built from a user-facing name, is rejected stating the rules and the fix + const string invalidKey = "trade_log_ai_hardware_&_cloud.csv"; + var errorMessage = string.Empty; + try + { + ObjectStore.Save(invalidKey, "a,b\n1,2"); + } + catch (ArgumentException exception) + { + errorMessage = exception.Message; + } + if (!errorMessage.Contains(Storage.ObjectStore.SupportedKeyRules) || !errorMessage.Contains("SanitizeKey")) + { + throw new RegressionTestException($"Expected the key rules in the unsupported key error, got: '{errorMessage}'"); + } + + // SanitizeKey converts the arbitrary name into a supported key + var sanitized = Storage.ObjectStore.SanitizeKey(invalidKey); + if (sanitized != "trade_log_ai_hardware___cloud.csv") + { + throw new RegressionTestException($"Unexpected sanitized key: '{sanitized}'"); + } + if (!Storage.ObjectStore.IsSupportedKey(sanitized) || !ObjectStore.Save(sanitized, "a,b\n1,2")) + { + throw new RegressionTestException("Expected the sanitized key to be storable"); + } + + // SaveText is an alias of Save + if (!ObjectStore.SaveText("ergonomics_report.txt", "The strategy went up") || + ObjectStore.Read("ergonomics_report.txt") != "The strategy went up") + { + throw new RegressionTestException("Expected the SaveText/Read round trip to succeed"); + } + + // reading a missing key lists the available keys in the error + errorMessage = string.Empty; + try + { + ObjectStore.ReadBytes("ergonomics_missing.json"); + } + catch (KeyNotFoundException exception) + { + errorMessage = exception.Message; + } + if (!errorMessage.Contains("Keys: [") || !errorMessage.Contains($"'{sanitized}'")) + { + throw new RegressionTestException($"Expected the available keys in the missing key error, got: '{errorMessage}'"); + } + + ObjectStore.Delete(sanitized); + ObjectStore.Delete("ergonomics_report.txt"); + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public List Languages { get; } = new() { Language.CSharp, Language.Python }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public long DataPoints => 6; + + /// + /// Data Points count of the algorithm history + /// + public int AlgorithmHistoryDataPoints => 0; + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "0"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "0%"}, + {"Drawdown", "0%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "100000"}, + {"Net Profit", "0%"}, + {"Sharpe Ratio", "0"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "0%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "0"}, + {"Beta", "0"}, + {"Annual Standard Deviation", "0"}, + {"Annual Variance", "0"}, + {"Information Ratio", "0"}, + {"Tracking Error", "0"}, + {"Treynor Ratio", "0"}, + {"Total Fees", "$0.00"}, + {"Estimated Strategy Capacity", "$0"}, + {"Lowest Capacity Asset", ""}, + {"Portfolio Turnover", "0%"}, + {"Drawdown Recovery", "0"}, + {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"} + }; + } +} diff --git a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py new file mode 100644 index 000000000000..fb0eaf184af2 --- /dev/null +++ b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py @@ -0,0 +1,104 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from AlgorithmImports import * +from decimal import Decimal + +### +### Regression algorithm asserting the ObjectStore ergonomics: unsupported keys are rejected with the +### key rules stated in the error, sanitize_key converts arbitrary names into supported keys, save_text +### is an alias of save, reading a missing key lists the available keys in the error, and the tolerant +### save_json/read_json and save_dataframe helpers handle datetime/date/Decimal/Symbol and data frames +### out of the box. +### +class ObjectStoreErgonomicsRegressionAlgorithm(QCAlgorithm): + + def initialize(self): + self.set_start_date(2013, 10, 7) + self.set_end_date(2013, 10, 11) + + self.set_benchmark(lambda x: 0) + + self.spy = self.add_equity("SPY", Resolution.DAILY).symbol + + def on_end_of_algorithm(self): + # an unsupported key, e.g. built from a user-facing name, is rejected stating the rules and the fix. + # note: BaseException because AlgorithmImports shadows the builtin Exception with System.Exception + invalid_key = "trade_log_ai_hardware_&_cloud.csv" + error = None + try: + self.object_store.save(invalid_key, "a,b\n1,2") + except BaseException as e: + error = str(e) + if error is None or "keys may only contain" not in error or "SanitizeKey" not in error: + raise AssertionError(f"Expected the key rules in the unsupported key error, got: '{error}'") + + # sanitize_key converts the arbitrary name into a supported key + sanitized = self.object_store.sanitize_key(invalid_key) + if sanitized != "trade_log_ai_hardware___cloud.csv": + raise AssertionError(f"Unexpected sanitized key: '{sanitized}'") + if not self.object_store.save(sanitized, "a,b\n1,2"): + raise AssertionError("Expected the sanitized key to be storable") + + # save_json tolerates datetime/date/Decimal/Symbol values and non-string dictionary keys + trade_log = { + "time": datetime(2013, 10, 11, 16, 0, 0), + "date": date(2013, 10, 11), + "qty": Decimal("1.5"), + "symbol": self.spy, + "fills": [{"price": Decimal("167.42"), self.spy: 10}] + } + if not self.object_store.save_json("ergonomics/trade_log.json", trade_log): + raise AssertionError("Expected save_json to succeed") + data = self.object_store.read_json("ergonomics/trade_log.json") + expected = { + "time": "2013-10-11T16:00:00", + "date": "2013-10-11", + "qty": 1.5, + "symbol": "SPY", + "fills": [{"price": 167.42, "SPY": 10}] + } + if data != expected: + raise AssertionError(f"Unexpected read_json round trip result: {data}") + + # read_json returns the given default when the key is missing + if self.object_store.read_json("ergonomics/missing.json") is not None: + raise AssertionError("Expected read_json of a missing key to return None") + if self.object_store.read_json("ergonomics/missing.json", {"warmed_up": False}) != {"warmed_up": False}: + raise AssertionError("Expected read_json of a missing key to return the given default") + + # save_dataframe stores a pandas DataFrame as CSV + frame = pd.DataFrame({"close": [167.42, 168.0]}, index=pd.to_datetime(["2013-10-10", "2013-10-11"])) + if not self.object_store.save_dataframe("ergonomics/history.csv", frame): + raise AssertionError("Expected save_dataframe to succeed") + csv = self.object_store.read("ergonomics/history.csv") + if "close" not in csv or "2013-10-10" not in csv: + raise AssertionError(f"Unexpected save_dataframe content: {csv}") + + # save_text is an alias of save + if not self.object_store.save_text("ergonomics/report.txt", "The strategy went up"): + raise AssertionError("Expected save_text to succeed") + if self.object_store.read("ergonomics/report.txt") != "The strategy went up": + raise AssertionError("Expected the save_text/read round trip to succeed") + + # reading a missing key lists the available keys in the error + error = None + try: + self.object_store.read_bytes("ergonomics/missing.json") + except BaseException as e: + error = str(e) + if error is None or "Keys: [" not in error or f"'{sanitized}'" not in error: + raise AssertionError(f"Expected the available keys in the missing key error, got: '{error}'") + + for key in [sanitized, "ergonomics/trade_log.json", "ergonomics/history.csv", "ergonomics/report.txt"]: + self.object_store.delete(key) diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index bb4795c00d60..d2b73bb7766a 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -17,9 +17,12 @@ using System.Collections; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; +using System.Text.RegularExpressions; using System.Xml.Serialization; using Newtonsoft.Json; +using Python.Runtime; using QuantConnect.Interfaces; using QuantConnect.Packets; @@ -30,6 +33,23 @@ namespace QuantConnect.Storage /// public class ObjectStore : IObjectStore { + // The single source of the key charset/extension rules, also enforced by the LocalObjectStore implementation + private static readonly Regex SupportedKeyRegex = new(@"^\.?[a-zA-Z0-9\\/_#\-\$= ]+\.?[a-zA-Z0-9]*$", RegexOptions.Compiled); + private static readonly Regex UnsupportedKeyCharactersRegex = new(@"[^a-zA-Z0-9\\/_#\-\$= .]", RegexOptions.Compiled); + private static readonly Regex KeyExtensionRegex = new(@"^[a-zA-Z0-9]+$", RegexOptions.Compiled); + + // Python helpers for the PyObject APIs, lazily created under the GIL + private static PyObject _jsonSerializeMethod; + private static PyObject _jsonDeserializeMethod; + private static PyObject _dataFrameSerializeMethod; + + /// + /// Human readable description of the format an object store key must follow, stated in key validation errors + /// + public static string SupportedKeyRules { get; } = "keys may only contain english letters, numbers, spaces and the characters" + + " '/', '\\', '_', '#', '-', '$' and '=', plus at most one '.' followed by a letters-and-numbers-only extension," + + " e.g. 'folder/trade_log-2024.csv'"; + /// /// Gets the maximum storage limit in bytes /// @@ -159,6 +179,57 @@ public string ReadString(string path, Encoding encoding = null) return Read(path, encoding); } + /// + /// Determines whether the given key follows the object store key format, see + /// + /// The object store key to validate + /// True if the key is supported + public static bool IsSupportedKey(string key) + { + return !string.IsNullOrEmpty(key) && SupportedKeyRegex.IsMatch(key) + // just in case + && key.Count(c => c == '/') <= 100 && key.Count(c => c == '\\') <= 100; + } + + /// + /// Converts an arbitrary name into a supported object store key by replacing every unsupported character + /// with '_', keeping at most the final '.extension'. Useful for programmatically-built keys, for instance + /// from user-facing names: 'trade log: AI & Cloud.csv' becomes 'trade log_ AI _ Cloud.csv' + /// + /// The arbitrary name to sanitize + /// A key following + public static string SanitizeKey(string key) + { + if (string.IsNullOrEmpty(key)) + { + throw new ArgumentException("ObjectStore.SanitizeKey(): key cannot be null or empty", nameof(key)); + } + if (IsSupportedKey(key)) + { + return key; + } + + var sanitized = UnsupportedKeyCharactersRegex.Replace(key, "_"); + + // a single trailing '.extension' of letters and numbers is allowed: keep the last dot if it + // introduces a valid extension and replace every other dot + var lastDot = sanitized.LastIndexOf('.'); + if (lastDot > 0 && KeyExtensionRegex.IsMatch(sanitized.Substring(lastDot + 1))) + { + sanitized = sanitized.Substring(0, lastDot).Replace('.', '_') + sanitized.Substring(lastDot); + } + else if (lastDot >= 0) + { + sanitized = sanitized.Replace('.', '_'); + } + + if (!IsSupportedKey(sanitized)) + { + throw new ArgumentException($"ObjectStore.SanitizeKey(): unable to sanitize key '{key}': object store {SupportedKeyRules}"); + } + return sanitized; + } + /// /// Returns the JSON deserialized object data for the specified path /// @@ -174,6 +245,28 @@ public T ReadJson(string path, Encoding encoding = null, JsonSerializerSettin return JsonConvert.DeserializeObject(json, settings); } + /// + /// Returns the JSON deserialized object data for the specified path as Python objects, + /// or the given default value if the key is not present + /// + /// The object path + /// Value to return when the key is not present. Defaults to None + /// The deserialized Python object, or if the key is not present + public PyObject ReadJson(string path, PyObject defaultValue = null) + { + if (!ContainsKey(path)) + { + return defaultValue; + } + var json = Read(path); + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var pyJson = json.ToPython(); + return _jsonDeserializeMethod.Invoke(pyJson); + } + } + /// /// Returns the XML deserialized object data for the specified path /// @@ -239,6 +332,19 @@ public bool SaveString(string path, string text, Encoding encoding = null) return _store.SaveBytes(path, encoding.GetBytes(text)); } + /// + /// Saves the object data in text format for the specified path. + /// Alias of + /// + /// The object path + /// The string object to be saved + /// The string encoding used, by default + /// True if the object was saved successfully + public bool SaveText(string path, string text, Encoding encoding = null) + { + return Save(path, text, encoding); + } + /// /// Saves the object data in JSON format for the specified path /// @@ -255,6 +361,47 @@ public bool SaveJson(string path, T obj, Encoding encoding = null, JsonSerial return SaveString(path, json, encoding); } + /// + /// Saves the given Python object in JSON format for the specified path, tolerating types the standard + /// json module rejects: datetime/date/time are stored in ISO-8601 format, Decimal and numpy scalars as + /// numbers and any other unsupported type (e.g. Symbol) as its string representation. Non-string + /// dictionary keys are stringified + /// + /// The object path + /// The Python object to be saved + /// True if the object was saved successfully + public bool SaveJson(string path, PyObject obj) + { + string json; + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var result = _jsonSerializeMethod.Invoke(obj); + json = result.As(); + } + return Save(path, json); + } + + /// + /// Saves the given pandas DataFrame (or Series) for the specified path, in JSON format if the + /// path has a '.json' extension and as CSV otherwise + /// + /// The object path + /// The pandas DataFrame or Series to be saved + /// True if the object was saved successfully + public bool SaveDataframe(string path, PyObject dataFrame) + { + string serialized; + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var pyPath = (path ?? string.Empty).ToPython(); + using var result = _dataFrameSerializeMethod.Invoke(dataFrame, pyPath); + serialized = result.As(); + } + return Save(path, serialized); + } + /// /// Saves the object data in XML format for the specified path /// @@ -298,5 +445,59 @@ public void Dispose() { _store.Dispose(); } + + /// + /// Lazily creates the Python helper methods backing the PyObject APIs. Must be called under the GIL + /// + private static void EnsurePythonHelpers() + { + if (_jsonSerializeMethod == null) + { + var module = PyModule.FromString("object_store_helpers", @"from json import dumps, loads +from datetime import datetime, date, time +from decimal import Decimal +try: + import numpy +except ImportError: + numpy = None + +def _default(value): + if isinstance(value, (datetime, date, time)): + return value.isoformat() + if isinstance(value, Decimal): + return float(value) + if numpy is not None and isinstance(value, numpy.generic): + return value.item() + # Symbol and any other type json does not handle + return str(value) + +def _normalize(value): + if isinstance(value, dict): + # json requires str/int/float/bool/None keys: stringify anything else (Symbol, datetime, ...) + return { (k if isinstance(k, (str, int, float, bool)) or k is None else str(_default(k))): _normalize(v) + for k, v in value.items() } + if isinstance(value, (list, tuple, set)): + return [_normalize(v) for v in value] + return value + +def serialize(value): + return dumps(_normalize(value), default=_default) + +def deserialize(json_string): + return loads(json_string) + +def serialize_dataframe(value, key): + if not hasattr(value, 'to_csv'): + raise TypeError(f'save_dataframe() expects a pandas DataFrame or Series but received {type(value).__name__}') + if key.lower().endswith('.json'): + return value.to_json() + return value.to_csv() +"); + _jsonDeserializeMethod = module.GetAttr("deserialize"); + _dataFrameSerializeMethod = module.GetAttr("serialize_dataframe"); + // last so partial initialization is never observed + _jsonSerializeMethod = module.GetAttr("serialize"); + } + } } } \ No newline at end of file diff --git a/Engine/Storage/LocalObjectStore.cs b/Engine/Storage/LocalObjectStore.cs index 80690b8a7531..600667fcd10a 100644 --- a/Engine/Storage/LocalObjectStore.cs +++ b/Engine/Storage/LocalObjectStore.cs @@ -19,7 +19,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text.RegularExpressions; using System.Threading; using QuantConnect.Configuration; using QuantConnect.Interfaces; @@ -112,7 +111,6 @@ private bool IsDirty } private Timer _persistenceTimer; - private Regex _pathRegex = new(@"^\.?[a-zA-Z0-9\\/_#\-\$= ]+\.?[a-zA-Z0-9]*$", RegexOptions.Compiled); private readonly ConcurrentDictionary _storage = new(); private readonly object _persistLock = new object(); @@ -292,7 +290,14 @@ public byte[] ReadBytes(string path) // Ensure we have the key, also takes care of null or improper access if (!ContainsKey(path)) { - throw new KeyNotFoundException($"Object with path '{path}' was not found in the current project. " + + // list the available keys so a near-miss (typo, wrong extension) is visible in the error itself + var keys = Keys; + var sample = string.Join(", ", keys.Take(10).Select(key => $"'{key}'")); + if (keys.Count > 10) + { + sample += ", ..."; + } + throw new KeyNotFoundException($"Object with path '{path}' was not found in the current project. Keys: [{sample}]. " + "Please use ObjectStore.ContainsKey(key) to check if an object exists before attempting to read." ); } @@ -326,14 +331,12 @@ public bool SaveBytes(string path, byte[] contents) { throw new InvalidOperationException($"LocalObjectStore.SaveBytes(): {NoWritePermissionsError}"); } - else if (!_pathRegex.IsMatch(path)) - { - throw new ArgumentException($"LocalObjectStore: path is not supported: '{path}'"); - } - else if (path.Count(c => c == '/') > 100 || path.Count(c => c == '\\') > 100) + else if (!ObjectStore.IsSupportedKey(path)) { - // just in case - throw new ArgumentException($"LocalObjectStore: path is not supported: '{path}'"); + // state the rules in the error: keys are often built from user-facing names and only fail here, + // frequently in OnEndOfAlgorithm after an otherwise green run + throw new ArgumentException($"LocalObjectStore: path is not supported: '{path}'. Object store {ObjectStore.SupportedKeyRules}." + + $" Use ObjectStore.{nameof(ObjectStore.SanitizeKey)}(name) to convert an arbitrary name into a supported path."); } // after we check the regex diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index 6451c116bda3..624ce2a8dad2 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -541,6 +541,104 @@ public void ThrowsKeyNotFoundException_WhenObjectStoreDoesNotContainKey() Assert.IsTrue(error.Message.Contains("Please use ObjectStore.ContainsKey(key)")); } + [Test] + public void KeyNotFoundErrorListsTheAvailableKeys() + { + // dedicated store instance: its in-memory keys are listed first in the error + using var store = new ObjectStore(new TestLocalObjectStore()); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.IsTrue(store.SaveBytes("available_key_a.txt", new byte[] { 1 })); + Assert.IsTrue(store.SaveBytes("available_key_b.txt", new byte[] { 1 })); + + var error = Assert.Throws(() => store.ReadBytes("missing.missing")); + + Assert.IsTrue(error.Message.Contains("Keys: ["), error.Message); + Assert.IsTrue(error.Message.Contains("'available_key_a.txt'"), error.Message); + Assert.IsTrue(error.Message.Contains("'available_key_b.txt'"), error.Message); + + store.Delete("available_key_a.txt"); + store.Delete("available_key_b.txt"); + } + + [Test] + public void KeyNotFoundErrorTruncatesTheAvailableKeysListAtTen() + { + using var store = new ObjectStore(new TestLocalObjectStore()); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + for (var i = 0; i < 11; i++) + { + Assert.IsTrue(store.SaveBytes($"truncation_key_{i}.txt", new byte[] { 1 })); + } + + var error = Assert.Throws(() => store.ReadBytes("missing.missing")); + + Assert.IsTrue(error.Message.Contains(", ...]"), error.Message); + + for (var i = 0; i < 11; i++) + { + store.Delete($"truncation_key_{i}.txt"); + } + } + + [TestCase("my_key", true)] + [TestCase("./my_key/nested\\file.csv", true)] + [TestCase("file with spaces #1-$=.txt", true)] + [TestCase("trade_log_ai_hardware_&_cloud.csv", false)] + [TestCase("file.tar.gz", false)] + [TestCase("file.name_v2", false)] + [TestCase("**abc**", false)] + [TestCase("", false)] + [TestCase(null, false)] + public void ValidatesKeys(string key, bool isSupported) + { + Assert.AreEqual(isSupported, ObjectStore.IsSupportedKey(key)); + } + + [Test] + public void UnsupportedKeyErrorStatesTheKeyRules() + { + var error = Assert.Throws( + () => _store.SaveBytes("trade_log_ai_hardware_&_cloud.csv", new byte[] { 1 })); + + Assert.IsTrue(error.Message.Contains("path is not supported: 'trade_log_ai_hardware_&_cloud.csv'"), error.Message); + Assert.IsTrue(error.Message.Contains(ObjectStore.SupportedKeyRules), error.Message); + Assert.IsTrue(error.Message.Contains("SanitizeKey"), error.Message); + } + + [TestCase("my_key/file.csv", "my_key/file.csv")] + [TestCase("trade_log_ai_hardware_&_cloud.csv", "trade_log_ai_hardware___cloud.csv")] + [TestCase("trade log: AI & Cloud.csv", "trade log_ AI _ Cloud.csv")] + [TestCase("file.tar.gz", "file_tar.gz")] + [TestCase("résumé.pdf", "r_sum_.pdf")] + [TestCase("a..", "a__")] + [TestCase("100%.json", "100_.json")] + public void SanitizeKeyReturnsAStorableKey(string key, string expectedSanitizedKey) + { + var sanitized = ObjectStore.SanitizeKey(key); + + Assert.AreEqual(expectedSanitizedKey, sanitized); + Assert.IsTrue(ObjectStore.IsSupportedKey(sanitized)); + Assert.IsTrue(_store.SaveBytes(sanitized, new byte[] { 1 })); + + _store.Delete(sanitized); + } + + [TestCase(null)] + [TestCase("")] + public void SanitizeKeyThrowsOnNullOrEmptyKey(string key) + { + Assert.Throws(() => ObjectStore.SanitizeKey(key)); + } + + [Test] + public void SaveTextIsAnAliasOfSave() + { + Assert.IsTrue(_store.SaveText("save_text_alias.txt", "abc")); + Assert.AreEqual("abc", _store.Read("save_text_alias.txt")); + + _store.Delete("save_text_alias.txt"); + } + [TestCase("my_key", "./LocalObjectStoreTests/my_key")] [TestCase("test/123", "./LocalObjectStoreTests/test/123")] [TestCase("**abc**", null)] @@ -1016,6 +1114,64 @@ def add_data(object_store): } } + [Test] + public void PythonSaveJsonReadJsonSaveDataframeAndSanitizeKey() + { + using (Py.GIL()) + { + var testModule = PyModule.FromString("TestObjectStoreErgonomics", + @" +from AlgorithmImports import * +from datetime import datetime +from decimal import Decimal + +def test(object_store): + symbol = Symbol.create('SPY', SecurityType.EQUITY, Market.USA) + + # save_json tolerates datetime/date/Decimal/Symbol values and non-string dictionary keys + object_store.save_json('ergonomics/log.json', { + 'time': datetime(2024, 1, 2, 3, 4, 5), + 'date': datetime(2024, 1, 1).date(), + 'qty': Decimal('1.5'), + 'symbol': symbol, + 'fills': [{symbol: 10}] + }) + data = object_store.read_json('ergonomics/log.json') + # symbols are stored as their string representation, which depends on the symbol cache state + assert data == { + 'time': '2024-01-02T03:04:05', + 'date': '2024-01-01', + 'qty': 1.5, + 'symbol': str(symbol), + 'fills': [{str(symbol): 10}] + }, data + + # read_json returns the given default when the key is missing + assert object_store.read_json('ergonomics/missing.json') is None + assert object_store.read_json('ergonomics/missing.json', {'a': 1}) == {'a': 1} + + # save_text is an alias of save + object_store.save_text('ergonomics/report.txt', 'hello') + assert object_store.read('ergonomics/report.txt') == 'hello' + + # save_dataframe stores a DataFrame as CSV + import pandas as pd + frame = pd.DataFrame({'a': [1, 2], 'b': [3.5, 4.5]}) + object_store.save_dataframe('ergonomics/frame.csv', frame) + assert 'a,b' in object_store.read('ergonomics/frame.csv') + + # sanitize_key is reachable from the instance + assert object_store.sanitize_key('trade_log_ai_hardware_&_cloud.csv') == 'trade_log_ai_hardware___cloud.csv' + + for key in ['ergonomics/log.json', 'ergonomics/report.txt', 'ergonomics/frame.csv']: + object_store.delete(key) +"); + + dynamic test = testModule.GetAttr("test"); + Assert.DoesNotThrow(() => test(_store)); + } + } + private static void DummyMachineLearning(string outputFile, string content) { try From 14a54132dd9de2caaad2fcb2b34eca0d6697e1ea Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 14 Oct 2026 10:08:00 -0400 Subject: [PATCH 02/14] Validate keys eagerly with per-issue hints, add ReadText and ReadDataframe - ObjectStore.ValidateKey replaces the public SupportedKeyRules and names the problem in the key: unsupported characters, extra dots or a bad extension - The Python ReadJson throws on a missing key like every other read method - ReadText and ReadDataframe mirror SaveText and SaveDataframe --- ...bjectStoreErgonomicsRegressionAlgorithm.cs | 32 +++- ...bjectStoreErgonomicsRegressionAlgorithm.py | 46 ++++-- Common/Storage/ObjectStore.cs | 145 +++++++++++++++--- Engine/Storage/LocalObjectStore.cs | 8 +- Tests/Common/Storage/LocalObjectStoreTests.cs | 58 +++++-- 5 files changed, 227 insertions(+), 62 deletions(-) diff --git a/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs b/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs index 05a86aed9a37..1c7235fd7dd0 100644 --- a/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs +++ b/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs @@ -22,9 +22,9 @@ namespace QuantConnect.Algorithm.CSharp /// /// Regression algorithm asserting the ObjectStore ergonomics: unsupported keys are rejected with the /// key rules stated in the error, converts arbitrary names - /// into supported keys, is an alias of Save and reading a - /// missing key lists the available keys in the error. The Python version of this algorithm additionally - /// covers the tolerant save_json/read_json and save_dataframe helpers + /// into supported keys, and + /// are aliases of Save and Read, and reading a missing key lists the available keys in the error. The Python + /// version of this algorithm additionally covers the tolerant save_json/read_json and save_dataframe/read_dataframe helpers /// public class ObjectStoreErgonomicsRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition { @@ -57,9 +57,24 @@ public override void OnEndOfAlgorithm() { errorMessage = exception.Message; } - if (!errorMessage.Contains(Storage.ObjectStore.SupportedKeyRules) || !errorMessage.Contains("SanitizeKey")) + if (!errorMessage.Contains("unsupported character '&'") || !errorMessage.Contains("keys may only contain") || + !errorMessage.Contains("SanitizeKey")) { - throw new RegressionTestException($"Expected the key rules in the unsupported key error, got: '{errorMessage}'"); + throw new RegressionTestException($"Expected the key issue and rules in the unsupported key error, got: '{errorMessage}'"); + } + // ValidateKey states the same error eagerly, when the key is built + errorMessage = string.Empty; + try + { + Storage.ObjectStore.ValidateKey(invalidKey); + } + catch (ArgumentException exception) + { + errorMessage = exception.Message; + } + if (!errorMessage.Contains("unsupported character '&'")) + { + throw new RegressionTestException($"Expected ValidateKey to reject the key stating the issue, got: '{errorMessage}'"); } // SanitizeKey converts the arbitrary name into a supported key @@ -73,11 +88,12 @@ public override void OnEndOfAlgorithm() throw new RegressionTestException("Expected the sanitized key to be storable"); } - // SaveText is an alias of Save + // SaveText and ReadText are aliases of Save and Read if (!ObjectStore.SaveText("ergonomics_report.txt", "The strategy went up") || - ObjectStore.Read("ergonomics_report.txt") != "The strategy went up") + ObjectStore.Read("ergonomics_report.txt") != "The strategy went up" || + ObjectStore.ReadText("ergonomics_report.txt") != "The strategy went up") { - throw new RegressionTestException("Expected the SaveText/Read round trip to succeed"); + throw new RegressionTestException("Expected the SaveText/ReadText round trip to succeed"); } // reading a missing key lists the available keys in the error diff --git a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py index fb0eaf184af2..2e1deb13a09c 100644 --- a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py +++ b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py @@ -12,14 +12,15 @@ # limitations under the License. from AlgorithmImports import * +from System.Collections.Generic import KeyNotFoundException from decimal import Decimal ### ### Regression algorithm asserting the ObjectStore ergonomics: unsupported keys are rejected with the -### key rules stated in the error, sanitize_key converts arbitrary names into supported keys, save_text -### is an alias of save, reading a missing key lists the available keys in the error, and the tolerant -### save_json/read_json and save_dataframe helpers handle datetime/date/Decimal/Symbol and data frames -### out of the box. +### key rules stated in the error, sanitize_key converts arbitrary names into supported keys, save_text and +### read_text are aliases of save and read, reading a missing key lists the available keys in the error, and +### the tolerant save_json/read_json and save_dataframe/read_dataframe helpers handle datetime/date/Decimal/Symbol +### and data frames out of the box. ### class ObjectStoreErgonomicsRegressionAlgorithm(QCAlgorithm): @@ -40,8 +41,17 @@ def on_end_of_algorithm(self): self.object_store.save(invalid_key, "a,b\n1,2") except BaseException as e: error = str(e) - if error is None or "keys may only contain" not in error or "SanitizeKey" not in error: - raise AssertionError(f"Expected the key rules in the unsupported key error, got: '{error}'") + if error is None or "unsupported character '&'" not in error or "keys may only contain" not in error or "SanitizeKey" not in error: + raise AssertionError(f"Expected the key issue and rules in the unsupported key error, got: '{error}'") + + # validate_key states the same error eagerly, when the key is built + error = None + try: + self.object_store.validate_key(invalid_key) + except BaseException as e: + error = str(e) + if error is None or "unsupported character '&'" not in error: + raise AssertionError(f"Expected validate_key to reject the key stating the issue, got: '{error}'") # sanitize_key converts the arbitrary name into a supported key sanitized = self.object_store.sanitize_key(invalid_key) @@ -71,25 +81,31 @@ def on_end_of_algorithm(self): if data != expected: raise AssertionError(f"Unexpected read_json round trip result: {data}") - # read_json returns the given default when the key is missing - if self.object_store.read_json("ergonomics/missing.json") is not None: - raise AssertionError("Expected read_json of a missing key to return None") - if self.object_store.read_json("ergonomics/missing.json", {"warmed_up": False}) != {"warmed_up": False}: - raise AssertionError("Expected read_json of a missing key to return the given default") + # read_json throws on a missing key like every other read method + error = None + try: + self.object_store.read_json("ergonomics/missing.json") + except KeyNotFoundException as e: + error = str(e) + if error is None: + raise AssertionError("Expected read_json of a missing key to throw") - # save_dataframe stores a pandas DataFrame as CSV + # save_dataframe stores a pandas DataFrame as CSV and read_dataframe restores it, time index included frame = pd.DataFrame({"close": [167.42, 168.0]}, index=pd.to_datetime(["2013-10-10", "2013-10-11"])) if not self.object_store.save_dataframe("ergonomics/history.csv", frame): raise AssertionError("Expected save_dataframe to succeed") csv = self.object_store.read("ergonomics/history.csv") if "close" not in csv or "2013-10-10" not in csv: raise AssertionError(f"Unexpected save_dataframe content: {csv}") + restored = self.object_store.read_dataframe("ergonomics/history.csv") + if not restored.equals(frame): + raise AssertionError(f"Unexpected read_dataframe round trip result: {restored}") - # save_text is an alias of save + # save_text/read_text are aliases of save/read if not self.object_store.save_text("ergonomics/report.txt", "The strategy went up"): raise AssertionError("Expected save_text to succeed") - if self.object_store.read("ergonomics/report.txt") != "The strategy went up": - raise AssertionError("Expected the save_text/read round trip to succeed") + if self.object_store.read_text("ergonomics/report.txt") != "The strategy went up": + raise AssertionError("Expected the save_text/read_text round trip to succeed") # reading a missing key lists the available keys in the error error = None diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index d2b73bb7766a..60c900020dd8 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -38,17 +38,15 @@ public class ObjectStore : IObjectStore private static readonly Regex UnsupportedKeyCharactersRegex = new(@"[^a-zA-Z0-9\\/_#\-\$= .]", RegexOptions.Compiled); private static readonly Regex KeyExtensionRegex = new(@"^[a-zA-Z0-9]+$", RegexOptions.Compiled); + private const string SupportedKeyRules = "keys may only contain english letters, numbers, spaces and the characters" + + " '/', '\\', '_', '#', '-', '$' and '=', plus at most one '.' followed by a letters-and-numbers-only extension," + + " e.g. 'folder/trade_log-2024.csv'"; + // Python helpers for the PyObject APIs, lazily created under the GIL private static PyObject _jsonSerializeMethod; private static PyObject _jsonDeserializeMethod; private static PyObject _dataFrameSerializeMethod; - - /// - /// Human readable description of the format an object store key must follow, stated in key validation errors - /// - public static string SupportedKeyRules { get; } = "keys may only contain english letters, numbers, spaces and the characters" + - " '/', '\\', '_', '#', '-', '$' and '=', plus at most one '.' followed by a letters-and-numbers-only extension," + - " e.g. 'folder/trade_log-2024.csv'"; + private static PyObject _dataFrameDeserializeMethod; /// /// Gets the maximum storage limit in bytes @@ -180,7 +178,20 @@ public string ReadString(string path, Encoding encoding = null) } /// - /// Determines whether the given key follows the object store key format, see + /// Returns the object data for the specified path as a string. Alias of + /// + /// The object path + /// The string encoding used, by default + /// A string containing the data + public string ReadText(string path, Encoding encoding = null) + { + return Read(path, encoding); + } + + /// + /// Determines whether the given key follows the object store key format: english letters, numbers, spaces + /// and the characters '/', '\', '_', '#', '-', '$' and '=', plus at most one '.' followed by a + /// letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv' /// /// The object store key to validate /// True if the key is supported @@ -191,13 +202,78 @@ public static bool IsSupportedKey(string key) && key.Count(c => c == '/') <= 100 && key.Count(c => c == '\\') <= 100; } + /// + /// Throws if the given key does not follow the object store key format, see . + /// Use it to validate programmatically-built keys when they are constructed, instead of at the final save + /// + /// The object store key to validate + /// The key is not supported. The message states the key rules + public static void ValidateKey(string key) + { + if (!IsSupportedKey(key)) + { + throw new ArgumentException($"ObjectStore: path is not supported: '{key}': {string.Join("; ", GetKeyIssues(key))}." + + $" Object store {SupportedKeyRules}. Use ObjectStore.{nameof(SanitizeKey)}(name) to convert an arbitrary name into a supported path."); + } + } + + /// + /// Describes why a key fails , one entry per problem found + /// + private static List GetKeyIssues(string key) + { + var issues = new List(); + if (string.IsNullOrEmpty(key)) + { + issues.Add("the key is empty"); + return issues; + } + + var unsupported = UnsupportedKeyCharactersRegex.Matches(key).Select(match => $"'{match.Value}'").Distinct().ToList(); + if (unsupported.Count > 0) + { + issues.Add($"unsupported character{(unsupported.Count > 1 ? "s" : "")} {string.Join(", ", unsupported)}"); + } + + // a single leading '.' is allowed, every other '.' must introduce the final extension + var name = key.TrimStart('.'); + var dots = name.Count(c => c == '.'); + if (dots > 1) + { + issues.Add($"it has {dots} '.' but at most one '.extension' is allowed"); + } + else if (dots == 1) + { + var extension = name.Substring(name.IndexOf('.') + 1); + if (extension.Length > 0 && !KeyExtensionRegex.IsMatch(extension)) + { + issues.Add($"the extension '{extension}' may only contain letters and numbers"); + } + } + if (name.Length == 0 || name[0] == '.') + { + issues.Add("it has no name before the '.'"); + } + + if (key.Count(c => c == '/') > 100 || key.Count(c => c == '\\') > 100) + { + issues.Add("it has more than 100 '/' or '\\' separators"); + } + + if (issues.Count == 0) + { + issues.Add("it does not follow the key format"); + } + return issues; + } + /// /// Converts an arbitrary name into a supported object store key by replacing every unsupported character /// with '_', keeping at most the final '.extension'. Useful for programmatically-built keys, for instance /// from user-facing names: 'trade log: AI & Cloud.csv' becomes 'trade log_ AI _ Cloud.csv' /// /// The arbitrary name to sanitize - /// A key following + /// A key following the object store key format, see public static string SanitizeKey(string key) { if (string.IsNullOrEmpty(key)) @@ -223,10 +299,7 @@ public static string SanitizeKey(string key) sanitized = sanitized.Replace('.', '_'); } - if (!IsSupportedKey(sanitized)) - { - throw new ArgumentException($"ObjectStore.SanitizeKey(): unable to sanitize key '{key}': object store {SupportedKeyRules}"); - } + ValidateKey(sanitized); return sanitized; } @@ -246,18 +319,12 @@ public T ReadJson(string path, Encoding encoding = null, JsonSerializerSettin } /// - /// Returns the JSON deserialized object data for the specified path as Python objects, - /// or the given default value if the key is not present + /// Returns the JSON deserialized object data for the specified path as Python objects /// /// The object path - /// Value to return when the key is not present. Defaults to None - /// The deserialized Python object, or if the key is not present - public PyObject ReadJson(string path, PyObject defaultValue = null) + /// The deserialized Python object + public PyObject ReadJson(string path) { - if (!ContainsKey(path)) - { - return defaultValue; - } var json = Read(path); using (Py.GIL()) { @@ -267,6 +334,25 @@ public PyObject ReadJson(string path, PyObject defaultValue = null) } } + /// + /// Returns the object data for the specified path as a pandas DataFrame, read as JSON if the path has + /// a '.json' extension and as CSV otherwise, so a round trip restores the frame. + /// A CSV index in ISO-8601 format is restored as a time index + /// + /// The object path + /// The pandas DataFrame + public PyObject ReadDataframe(string path) + { + var serialized = Read(path); + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var pySerialized = serialized.ToPython(); + using var pyPath = path.ToPython(); + return _dataFrameDeserializeMethod.Invoke(pySerialized, pyPath); + } + } + /// /// Returns the XML deserialized object data for the specified path /// @@ -492,9 +578,24 @@ raise TypeError(f'save_dataframe() expects a pandas DataFrame or Series but rece if key.lower().endswith('.json'): return value.to_json() return value.to_csv() + +def deserialize_dataframe(serialized, key): + import pandas + from io import StringIO + if key.lower().endswith('.json'): + return pandas.read_json(StringIO(serialized)) + # to_csv() writes the index as the first column, datetimes in ISO-8601: restore a time index when it parses as one + frame = pandas.read_csv(StringIO(serialized), index_col=0) + if pandas.api.types.is_string_dtype(frame.index): + try: + frame.index = pandas.to_datetime(frame.index, format='ISO8601') + except (ValueError, TypeError): + pass + return frame "); _jsonDeserializeMethod = module.GetAttr("deserialize"); _dataFrameSerializeMethod = module.GetAttr("serialize_dataframe"); + _dataFrameDeserializeMethod = module.GetAttr("deserialize_dataframe"); // last so partial initialization is never observed _jsonSerializeMethod = module.GetAttr("serialize"); } diff --git a/Engine/Storage/LocalObjectStore.cs b/Engine/Storage/LocalObjectStore.cs index 600667fcd10a..679ef52f414c 100644 --- a/Engine/Storage/LocalObjectStore.cs +++ b/Engine/Storage/LocalObjectStore.cs @@ -331,13 +331,7 @@ public bool SaveBytes(string path, byte[] contents) { throw new InvalidOperationException($"LocalObjectStore.SaveBytes(): {NoWritePermissionsError}"); } - else if (!ObjectStore.IsSupportedKey(path)) - { - // state the rules in the error: keys are often built from user-facing names and only fail here, - // frequently in OnEndOfAlgorithm after an otherwise green run - throw new ArgumentException($"LocalObjectStore: path is not supported: '{path}'. Object store {ObjectStore.SupportedKeyRules}." + - $" Use ObjectStore.{nameof(ObjectStore.SanitizeKey)}(name) to convert an arbitrary name into a supported path."); - } + ObjectStore.ValidateKey(path); // after we check the regex path = NormalizePath(path); diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index 624ce2a8dad2..2741d1aa5bd0 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -592,6 +592,14 @@ public void KeyNotFoundErrorTruncatesTheAvailableKeysListAtTen() public void ValidatesKeys(string key, bool isSupported) { Assert.AreEqual(isSupported, ObjectStore.IsSupportedKey(key)); + if (isSupported) + { + Assert.DoesNotThrow(() => ObjectStore.ValidateKey(key)); + } + else + { + Assert.Throws(() => ObjectStore.ValidateKey(key)); + } } [Test] @@ -600,11 +608,27 @@ public void UnsupportedKeyErrorStatesTheKeyRules() var error = Assert.Throws( () => _store.SaveBytes("trade_log_ai_hardware_&_cloud.csv", new byte[] { 1 })); - Assert.IsTrue(error.Message.Contains("path is not supported: 'trade_log_ai_hardware_&_cloud.csv'"), error.Message); - Assert.IsTrue(error.Message.Contains(ObjectStore.SupportedKeyRules), error.Message); + Assert.IsTrue(error.Message.Contains("path is not supported: 'trade_log_ai_hardware_&_cloud.csv': unsupported character '&'."), error.Message); + Assert.IsTrue(error.Message.Contains("keys may only contain english letters, numbers, spaces and the characters"), error.Message); Assert.IsTrue(error.Message.Contains("SanitizeKey"), error.Message); } + [TestCase("trade log: AI & Cloud.csv", "unsupported characters ':', '&'")] + [TestCase("résumé.pdf", "unsupported character 'é'")] + [TestCase("file.tar.gz", "it has 2 '.' but at most one '.extension' is allowed")] + [TestCase("file.name_v2", "the extension 'name_v2' may only contain letters and numbers")] + [TestCase("folder.v1/file", "the extension 'v1/file' may only contain letters and numbers")] + [TestCase("trade log: AI.v2.csv", "unsupported character ':'; it has 2 '.' but at most one '.extension' is allowed")] + [TestCase(".", "it has no name before the '.'")] + [TestCase("", "the key is empty")] + [TestCase(null, "the key is empty")] + public void UnsupportedKeyErrorStatesTheIssue(string key, string expectedIssue) + { + var error = Assert.Throws(() => ObjectStore.ValidateKey(key)); + + Assert.IsTrue(error.Message.Contains($"'{key}': {expectedIssue}."), error.Message); + } + [TestCase("my_key/file.csv", "my_key/file.csv")] [TestCase("trade_log_ai_hardware_&_cloud.csv", "trade_log_ai_hardware___cloud.csv")] [TestCase("trade log: AI & Cloud.csv", "trade log_ AI _ Cloud.csv")] @@ -631,10 +655,11 @@ public void SanitizeKeyThrowsOnNullOrEmptyKey(string key) } [Test] - public void SaveTextIsAnAliasOfSave() + public void SaveTextAndReadTextAreAliasesOfSaveAndRead() { Assert.IsTrue(_store.SaveText("save_text_alias.txt", "abc")); Assert.AreEqual("abc", _store.Read("save_text_alias.txt")); + Assert.AreEqual("abc", _store.ReadText("save_text_alias.txt")); _store.Delete("save_text_alias.txt"); } @@ -1115,16 +1140,24 @@ def add_data(object_store): } [Test] - public void PythonSaveJsonReadJsonSaveDataframeAndSanitizeKey() + public void PythonSaveJsonReadJsonSaveDataframeReadDataframeAndSanitizeKey() { using (Py.GIL()) { var testModule = PyModule.FromString("TestObjectStoreErgonomics", @" from AlgorithmImports import * +from System.Collections.Generic import KeyNotFoundException from datetime import datetime from decimal import Decimal +def assert_raises_key_not_found(action): + try: + action() + except KeyNotFoundException: + return + raise AssertionError('Expected a KeyNotFoundException') + def test(object_store): symbol = Symbol.create('SPY', SecurityType.EQUITY, Market.USA) @@ -1146,24 +1179,29 @@ def test(object_store): 'fills': [{str(symbol): 10}] }, data - # read_json returns the given default when the key is missing - assert object_store.read_json('ergonomics/missing.json') is None - assert object_store.read_json('ergonomics/missing.json', {'a': 1}) == {'a': 1} + # read_json throws on a missing key like every other read method + assert_raises_key_not_found(lambda: object_store.read_json('ergonomics/missing.json')) - # save_text is an alias of save + # save_text/read_text are aliases of save/read object_store.save_text('ergonomics/report.txt', 'hello') assert object_store.read('ergonomics/report.txt') == 'hello' + assert object_store.read_text('ergonomics/report.txt') == 'hello' - # save_dataframe stores a DataFrame as CSV + # save_dataframe stores a DataFrame as CSV, or JSON for '.json' keys, and read_dataframe restores it import pandas as pd frame = pd.DataFrame({'a': [1, 2], 'b': [3.5, 4.5]}) object_store.save_dataframe('ergonomics/frame.csv', frame) assert 'a,b' in object_store.read('ergonomics/frame.csv') + pd.testing.assert_frame_equal(object_store.read_dataframe('ergonomics/frame.csv'), frame) + object_store.save_dataframe('ergonomics/frame.json', frame) + assert object_store.read('ergonomics/frame.json') == frame.to_json() + pd.testing.assert_frame_equal(object_store.read_dataframe('ergonomics/frame.json'), frame) + assert_raises_key_not_found(lambda: object_store.read_dataframe('ergonomics/missing.csv')) # sanitize_key is reachable from the instance assert object_store.sanitize_key('trade_log_ai_hardware_&_cloud.csv') == 'trade_log_ai_hardware___cloud.csv' - for key in ['ergonomics/log.json', 'ergonomics/report.txt', 'ergonomics/frame.csv']: + for key in ['ergonomics/log.json', 'ergonomics/report.txt', 'ergonomics/frame.csv', 'ergonomics/frame.json']: object_store.delete(key) "); From e9561727cfc9f6bdc39ccb6a352b3c5653dc45f3 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 14 Oct 2026 10:11:16 -0400 Subject: [PATCH 03/14] Split the Python ObjectStore helper test into one test per method pair --- Tests/Common/Storage/LocalObjectStoreTests.cs | 118 ++++++++++++++---- 1 file changed, 92 insertions(+), 26 deletions(-) diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index 2741d1aa5bd0..94d48030da85 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -1140,28 +1140,13 @@ def add_data(object_store): } [Test] - public void PythonSaveJsonReadJsonSaveDataframeReadDataframeAndSanitizeKey() + public void PythonSaveJsonAndReadJsonRoundTripTolerantTypes() { - using (Py.GIL()) - { - var testModule = PyModule.FromString("TestObjectStoreErgonomics", - @" -from AlgorithmImports import * -from System.Collections.Generic import KeyNotFoundException -from datetime import datetime -from decimal import Decimal - -def assert_raises_key_not_found(action): - try: - action() - except KeyNotFoundException: - return - raise AssertionError('Expected a KeyNotFoundException') - + RunPythonTest(@" def test(object_store): symbol = Symbol.create('SPY', SecurityType.EQUITY, Market.USA) - # save_json tolerates datetime/date/Decimal/Symbol values and non-string dictionary keys + # datetime/date/Decimal/Symbol values and non-string dictionary keys, which json.dumps rejects object_store.save_json('ergonomics/log.json', { 'time': datetime(2024, 1, 2, 3, 4, 5), 'date': datetime(2024, 1, 1).date(), @@ -1179,31 +1164,112 @@ def test(object_store): 'fills': [{str(symbol): 10}] }, data - # read_json throws on a missing key like every other read method + object_store.delete('ergonomics/log.json') +"); + } + + [Test] + public void PythonReadJsonThrowsOnMissingKey() + { + RunPythonTest(@" +def test(object_store): assert_raises_key_not_found(lambda: object_store.read_json('ergonomics/missing.json')) +"); + } - # save_text/read_text are aliases of save/read + [Test] + public void PythonSaveTextAndReadTextAreAliasesOfSaveAndRead() + { + RunPythonTest(@" +def test(object_store): object_store.save_text('ergonomics/report.txt', 'hello') assert object_store.read('ergonomics/report.txt') == 'hello' assert object_store.read_text('ergonomics/report.txt') == 'hello' - # save_dataframe stores a DataFrame as CSV, or JSON for '.json' keys, and read_dataframe restores it - import pandas as pd + object_store.delete('ergonomics/report.txt') +"); + } + + [Test] + public void PythonSaveDataframeAndReadDataframeRoundTripAsCsv() + { + RunPythonTest(@" +def test(object_store): frame = pd.DataFrame({'a': [1, 2], 'b': [3.5, 4.5]}) object_store.save_dataframe('ergonomics/frame.csv', frame) assert 'a,b' in object_store.read('ergonomics/frame.csv') pd.testing.assert_frame_equal(object_store.read_dataframe('ergonomics/frame.csv'), frame) + + object_store.delete('ergonomics/frame.csv') +"); + } + + [Test] + public void PythonSaveDataframeAndReadDataframeRoundTripAsJson() + { + RunPythonTest(@" +def test(object_store): + frame = pd.DataFrame({'a': [1, 2], 'b': [3.5, 4.5]}) object_store.save_dataframe('ergonomics/frame.json', frame) assert object_store.read('ergonomics/frame.json') == frame.to_json() pd.testing.assert_frame_equal(object_store.read_dataframe('ergonomics/frame.json'), frame) + + object_store.delete('ergonomics/frame.json') +"); + } + + [Test] + public void PythonReadDataframeRestoresATimeIndex() + { + RunPythonTest(@" +def test(object_store): + frame = pd.DataFrame({'close': [167.42, 168.0]}, index=pd.to_datetime(['2013-10-10', '2013-10-11'])) + object_store.save_dataframe('ergonomics/history.csv', frame) + pd.testing.assert_frame_equal(object_store.read_dataframe('ergonomics/history.csv'), frame) + + object_store.delete('ergonomics/history.csv') +"); + } + + [Test] + public void PythonReadDataframeThrowsOnMissingKey() + { + RunPythonTest(@" +def test(object_store): assert_raises_key_not_found(lambda: object_store.read_dataframe('ergonomics/missing.csv')) +"); + } - # sanitize_key is reachable from the instance + [Test] + public void PythonSanitizeKeyIsReachableFromTheInstance() + { + RunPythonTest(@" +def test(object_store): assert object_store.sanitize_key('trade_log_ai_hardware_&_cloud.csv') == 'trade_log_ai_hardware___cloud.csv' - - for key in ['ergonomics/log.json', 'ergonomics/report.txt', 'ergonomics/frame.csv', 'ergonomics/frame.json']: - object_store.delete(key) "); + } + + /// + /// Runs the 'test(object_store)' function of the given Python source against the test store + /// + private void RunPythonTest(string source) + { + using (Py.GIL()) + { + var testModule = PyModule.FromString("TestObjectStoreErgonomics", @" +from AlgorithmImports import * +from System.Collections.Generic import KeyNotFoundException +from datetime import datetime +from decimal import Decimal +import pandas as pd + +def assert_raises_key_not_found(action): + try: + action() + except KeyNotFoundException: + return + raise AssertionError('Expected a KeyNotFoundException') +" + source); dynamic test = testModule.GetAttr("test"); Assert.DoesNotThrow(() => test(_store)); From 2405d2c0816443e870942f744cca19a757e82be9 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 30 Oct 2026 17:43:10 -0400 Subject: [PATCH 04/14] Serialize only faithful types in the Python SaveJson, document lossy SanitizeKey - The Python helper is an allow-list: numpy arrays via tolist, NaN and infinity as null, .NET objects through Newtonsoft like SaveJson, pandas rejected pointing to SaveDataframe, other types raise TypeError - Top-level Python scalars are documented as not accepted - SanitizeKey is documented as lossy, with a collision test - ReadDataframe documents the Series round trip --- ...bjectStoreErgonomicsRegressionAlgorithm.py | 26 ++- Common/Storage/ObjectStore.cs | 81 ++++++--- Tests/Common/Storage/LocalObjectStoreTests.cs | 161 +++++++++++++++++- 3 files changed, 235 insertions(+), 33 deletions(-) diff --git a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py index 2e1deb13a09c..d76091bbf12f 100644 --- a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py +++ b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py @@ -12,7 +12,7 @@ # limitations under the License. from AlgorithmImports import * -from System.Collections.Generic import KeyNotFoundException +from System.Collections.Generic import KeyNotFoundException, List from decimal import Decimal ### @@ -60,13 +60,15 @@ def on_end_of_algorithm(self): if not self.object_store.save(sanitized, "a,b\n1,2"): raise AssertionError("Expected the sanitized key to be storable") - # save_json tolerates datetime/date/Decimal/Symbol values and non-string dictionary keys + # save_json tolerates datetime/date/Decimal/numpy/NaN values, .NET objects and non-string dictionary keys trade_log = { "time": datetime(2013, 10, 11, 16, 0, 0), "date": date(2013, 10, 11), "qty": Decimal("1.5"), "symbol": self.spy, - "fills": [{"price": Decimal("167.42"), self.spy: 10}] + "fills": [{"price": Decimal("167.42"), self.spy: 10}], + "fill_quantities": List[int]([10, -10]), + "returns": np.array([0.01, np.nan]) } if not self.object_store.save_json("ergonomics/trade_log.json", trade_log): raise AssertionError("Expected save_json to succeed") @@ -75,11 +77,25 @@ def on_end_of_algorithm(self): "time": "2013-10-11T16:00:00", "date": "2013-10-11", "qty": 1.5, - "symbol": "SPY", - "fills": [{"price": 167.42, "SPY": 10}] + "symbol": {"value": "SPY", "id": str(self.spy.id), "permtick": "SPY"}, + "fills": [{"price": 167.42, "SPY": 10}], + "fill_quantities": [10, -10], + "returns": [0.01, None] } if data != expected: raise AssertionError(f"Unexpected read_json round trip result: {data}") + # the symbol id and the ticker key both resolve back to the added security through the symbol cache + if self.symbol(data["symbol"]["id"]) != self.spy or self.symbol("SPY") != self.spy: + raise AssertionError(f"Expected the stored symbol to resolve to {self.spy}, got: {data['symbol']}") + + # save_json refuses what it cannot serialize faithfully instead of storing its string + error = None + try: + self.object_store.save_json("ergonomics/rejected.json", {"history": pd.Series([1.0, 2.0])}) + except TypeError as e: + error = str(e) + if error is None or "save_dataframe" not in error: + raise AssertionError(f"Expected save_json to reject a pandas Series pointing to save_dataframe, got: '{error}'") # read_json throws on a missing key like every other read method error = None diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index 60c900020dd8..843d23c875ad 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -270,7 +270,10 @@ private static List GetKeyIssues(string key) /// /// Converts an arbitrary name into a supported object store key by replacing every unsupported character /// with '_', keeping at most the final '.extension'. Useful for programmatically-built keys, for instance - /// from user-facing names: 'trade log: AI & Cloud.csv' becomes 'trade log_ AI _ Cloud.csv' + /// from user-facing names: 'trade log: AI & Cloud.csv' becomes 'trade log_ AI _ Cloud.csv'. + /// The mapping is lossy: distinct names can sanitize to the same key, e.g. 'a&b.csv' and 'a:b.csv' both + /// become 'a_b.csv', so it only sanitizes, it does not generate unique keys. Checking for collisions, + /// for instance with , is up to the caller /// /// The arbitrary name to sanitize /// A key following the object store key format, see @@ -337,7 +340,8 @@ public PyObject ReadJson(string path) /// /// Returns the object data for the specified path as a pandas DataFrame, read as JSON if the path has /// a '.json' extension and as CSV otherwise, so a round trip restores the frame. - /// A CSV index in ISO-8601 format is restored as a time index + /// A CSV index in ISO-8601 format is restored as a time index. A saved Series comes back as a + /// one-column DataFrame, call squeeze() on it to get the Series /// /// The object path /// The pandas DataFrame @@ -448,14 +452,18 @@ public bool SaveJson(string path, T obj, Encoding encoding = null, JsonSerial } /// - /// Saves the given Python object in JSON format for the specified path, tolerating types the standard - /// json module rejects: datetime/date/time are stored in ISO-8601 format, Decimal and numpy scalars as - /// numbers and any other unsupported type (e.g. Symbol) as its string representation. Non-string - /// dictionary keys are stringified + /// Saves the given Python dictionary, list, tuple or .NET object in JSON format for the specified path, + /// tolerating nested types the standard json module rejects: datetime/date/time are stored in ISO-8601 + /// format, Decimal and numpy values as numbers, NaN and infinity as null, .NET objects (Symbol included) + /// as the C# overload writes them and non-string dictionary keys as strings. + /// pandas objects are rejected, use , and so is any other type that would + /// not round trip. A top-level Python str, int, float or bool is not accepted, use + /// for text /// /// The object path /// The Python object to be saved /// True if the object was saved successfully + /// A TypeError naming the value that cannot be serialized public bool SaveJson(string path, PyObject obj) { string json; @@ -542,32 +550,53 @@ private static void EnsurePythonHelpers() var module = PyModule.FromString("object_store_helpers", @"from json import dumps, loads from datetime import datetime, date, time from decimal import Decimal -try: - import numpy -except ImportError: - numpy = None - -def _default(value): - if isinstance(value, (datetime, date, time)): - return value.isoformat() - if isinstance(value, Decimal): - return float(value) - if numpy is not None and isinstance(value, numpy.generic): - return value.item() - # Symbol and any other type json does not handle - return str(value) +from math import isfinite +import numpy +import pandas +import System +import Newtonsoft.Json as newtonsoft + +def _key(key): + # json only accepts str/int/float/bool/None keys; a .NET key is its string, like a .NET dictionary key + if isinstance(key, (str, int, float)) or key is None: + return key + if isinstance(key, (datetime, date, time)): + return key.isoformat() + if isinstance(key, Decimal): + return str(key) + if isinstance(key, numpy.generic): + return key.item() + if isinstance(key, System.Object): + return str(key) + raise TypeError(f'save_json() cannot use a {type(key).__name__} as a dictionary key') def _normalize(value): if isinstance(value, dict): - # json requires str/int/float/bool/None keys: stringify anything else (Symbol, datetime, ...) - return { (k if isinstance(k, (str, int, float, bool)) or k is None else str(_default(k))): _normalize(v) - for k, v in value.items() } - if isinstance(value, (list, tuple, set)): + return {_key(k): _normalize(v) for k, v in value.items()} + if isinstance(value, (list, tuple, set, frozenset)): return [_normalize(v) for v in value] - return value + if isinstance(value, (str, int)) or value is None: + return value + if isinstance(value, float): + # NaN and infinity are not valid JSON + return value if isfinite(value) else None + if isinstance(value, (datetime, date, time)): + return value.isoformat() + if isinstance(value, Decimal): + return _normalize(float(value)) + if isinstance(value, numpy.ndarray): + return _normalize(value.tolist()) + if isinstance(value, numpy.generic): + return _normalize(value.item()) + if isinstance(value, (pandas.DataFrame, pandas.Series)): + raise TypeError(f'save_json() does not accept a pandas {type(value).__name__}, use save_dataframe() instead') + if isinstance(value, System.Object): + # .NET objects serialize like the C# SaveJson does + return loads(newtonsoft.JsonConvert.SerializeObject(value)) + raise TypeError(f'save_json() cannot serialize a {type(value).__name__}: convert it to a dict, list or str first') def serialize(value): - return dumps(_normalize(value), default=_default) + return dumps(_normalize(value), allow_nan=False) def deserialize(json_string): return loads(json_string) diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index 94d48030da85..edf67758fd27 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -654,6 +654,24 @@ public void SanitizeKeyThrowsOnNullOrEmptyKey(string key) Assert.Throws(() => ObjectStore.SanitizeKey(key)); } + [Test] + public void SanitizeKeyIsLossyAndDistinctNamesCanCollide() + { + var first = ObjectStore.SanitizeKey("trade_log_ai&cloud.csv"); + var second = ObjectStore.SanitizeKey("trade_log_ai:cloud.csv"); + + Assert.AreEqual("trade_log_ai_cloud.csv", first); + Assert.AreEqual(first, second); + Assert.AreEqual(first, ObjectStore.SanitizeKey("trade_log_ai_cloud.csv")); + + // the second save silently overwrites the first: collision checks are up to the caller + Assert.IsTrue(_store.Save(first, "first")); + Assert.IsTrue(_store.Save(second, "second")); + Assert.AreEqual("second", _store.Read(first)); + + _store.Delete(first); + } + [Test] public void SaveTextAndReadTextAreAliasesOfSaveAndRead() { @@ -1155,19 +1173,157 @@ def test(object_store): 'fills': [{symbol: 10}] }) data = object_store.read_json('ergonomics/log.json') - # symbols are stored as their string representation, which depends on the symbol cache state + # a Symbol value is written like the C# SaveJson does, a Symbol key as its string like a .NET dictionary key assert data == { 'time': '2024-01-02T03:04:05', 'date': '2024-01-01', 'qty': 1.5, - 'symbol': str(symbol), + 'symbol': {'value': 'SPY', 'id': str(symbol.id), 'permtick': 'SPY'}, 'fills': [{str(symbol): 10}] }, data + # both forms resolve back to the Symbol through the symbol cache, which is what QCAlgorithm.symbol(str) does + assert SymbolCache.get_symbol(data['symbol']['id']) == symbol + assert SymbolCache.get_symbol(next(iter(data['fills'][0]))) == symbol object_store.delete('ergonomics/log.json') "); } + [Test] + public void PythonSaveJsonSerializesDotNetObjectsLikeTheCSharpOverload() + { + RunPythonTest(@" +from System.Collections.Generic import List, Dictionary +import Newtonsoft.Json as newtonsoft + +def test(object_store): + numbers = List[int]([1, 2]) + weights = Dictionary[str, float]() + weights['a'] = 1.5 + point = IndicatorDataPoint(datetime(2020, 1, 1), 5) + + object_store.save_json('ergonomics/dotnet.json', {'numbers': numbers, 'weights': weights, 'point': point}) + data = object_store.read_json('ergonomics/dotnet.json') + assert data == { + 'numbers': [1, 2], + 'weights': {'a': 1.5}, + 'point': json.loads(newtonsoft.JsonConvert.serialize_object(point)) + }, data + + object_store.delete('ergonomics/dotnet.json') +"); + } + + [Test] + public void PythonSaveJsonConvertsNumpyArraysAndScalars() + { + RunPythonTest(@" +import numpy as np + +def test(object_store): + object_store.save_json('ergonomics/numpy.json', { + 'weights': np.array([1.5, 2.5]), + 'grid': np.array([[1, 2], [3, 4]]), + 'count': np.int64(3), + 'ratio': np.float32(0.5) + }) + data = object_store.read_json('ergonomics/numpy.json') + assert data == {'weights': [1.5, 2.5], 'grid': [[1, 2], [3, 4]], 'count': 3, 'ratio': 0.5}, data + + object_store.delete('ergonomics/numpy.json') +"); + } + + [Test] + public void PythonSaveJsonWritesNanAndInfinityAsNull() + { + RunPythonTest(@" +import numpy as np + +def test(object_store): + object_store.save_json('ergonomics/nan.json', { + 'pnl': float('nan'), + 'ratio': float('inf'), + 'drawdown': -float('inf'), + 'series': np.array([1.0, np.nan]), + 'scalar': np.float64('nan') + }) + text = object_store.read('ergonomics/nan.json') + assert 'NaN' not in text and 'Infinity' not in text, text + data = object_store.read_json('ergonomics/nan.json') + assert data == {'pnl': None, 'ratio': None, 'drawdown': None, 'series': [1.0, None], 'scalar': None}, data + + object_store.delete('ergonomics/nan.json') +"); + } + + [Test] + public void PythonSaveJsonRejectsPandasObjectsPointingToSaveDataframe() + { + RunPythonTest(@" +def test(object_store): + for value in [pd.DataFrame({'a': [1, 2]}), pd.Series([1, 2])]: + error = None + try: + object_store.save_json('ergonomics/pandas.json', {'data': value}) + except TypeError as e: + error = str(e) + assert error is not None and 'save_dataframe' in error and type(value).__name__ in error, error + assert not object_store.contains_key('ergonomics/pandas.json') +"); + } + + [Test] + public void PythonSaveJsonRejectsUnsupportedTypesNamingTheType() + { + RunPythonTest(@" +class TradeRecord: + pass + +def test(object_store): + for value in [TradeRecord(), object(), (x for x in [1])]: + error = None + try: + object_store.save_json('ergonomics/unsupported.json', {'data': value}) + except TypeError as e: + error = str(e) + assert error is not None and type(value).__name__ in error, error + assert not object_store.contains_key('ergonomics/unsupported.json') +"); + } + + [Test] + public void PythonSaveJsonRoundTripsTopLevelContainersAndRejectsTopLevelScalars() + { + RunPythonTest(@" +def test(object_store): + failures = [] + try: + for value in [None, [1, 'a'], (1, 2)]: + expected = list(value) if isinstance(value, tuple) else value + object_store.save_json('ergonomics/container.json', value) + data = object_store.read_json('ergonomics/container.json') + if data != expected: + failures.append(f'{value!r} -> {data!r}') + + # a top-level str/int/float/bool does not bind to the PyObject overload: the error names the mismatch + for value in ['text', 42, 1.5, True]: + try: + object_store.save_json('ergonomics/scalar.json', value) + failures.append(f'{value!r} was accepted') + except TypeError as e: + if 'No method matches' not in str(e): + failures.append(f'{value!r} -> {e}') + if object_store.contains_key('ergonomics/scalar.json'): + failures.append('a rejected scalar was saved') + finally: + for key in ['ergonomics/container.json', 'ergonomics/scalar.json']: + if object_store.contains_key(key): + object_store.delete(key) + assert not failures, failures +"); + } + [Test] public void PythonReadJsonThrowsOnMissingKey() { @@ -1261,6 +1417,7 @@ from AlgorithmImports import * from System.Collections.Generic import KeyNotFoundException from datetime import datetime from decimal import Decimal +import json import pandas as pd def assert_raises_key_not_found(action): From f44c2fc5b60df608766842b75167c81ebda8f154 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 30 Oct 2026 18:11:11 -0400 Subject: [PATCH 05/14] Save numpy datetimes and durations as ISO-8601 in the Python SaveJson, NaT as null - datetime64 scalars and arrays convert through pandas.Timestamp, so nanosecond values no longer save as integers - timedelta, pd.Timedelta and timedelta64 save as ISO-8601 durations - pd.NaT and NaT values save as null like NaN - a NaN dictionary key raises TypeError like the other rejections --- ...bjectStoreErgonomicsRegressionAlgorithm.py | 6 +- Common/Storage/ObjectStore.cs | 37 +++++---- Tests/Common/Storage/LocalObjectStoreTests.cs | 77 +++++++++++++++++++ 3 files changed, 103 insertions(+), 17 deletions(-) diff --git a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py index d76091bbf12f..b7d8ef6e9aeb 100644 --- a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py +++ b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py @@ -68,7 +68,8 @@ def on_end_of_algorithm(self): "symbol": self.spy, "fills": [{"price": Decimal("167.42"), self.spy: 10}], "fill_quantities": List[int]([10, -10]), - "returns": np.array([0.01, np.nan]) + "returns": np.array([0.01, np.nan]), + "bar_times": pd.DatetimeIndex([datetime(2013, 10, 10), datetime(2013, 10, 11)]).values } if not self.object_store.save_json("ergonomics/trade_log.json", trade_log): raise AssertionError("Expected save_json to succeed") @@ -80,7 +81,8 @@ def on_end_of_algorithm(self): "symbol": {"value": "SPY", "id": str(self.spy.id), "permtick": "SPY"}, "fills": [{"price": 167.42, "SPY": 10}], "fill_quantities": [10, -10], - "returns": [0.01, None] + "returns": [0.01, None], + "bar_times": ["2013-10-10T00:00:00", "2013-10-11T00:00:00"] } if data != expected: raise AssertionError(f"Unexpected read_json round trip result: {data}") diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index 843d23c875ad..ac8dcd15ee85 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -453,12 +453,13 @@ public bool SaveJson(string path, T obj, Encoding encoding = null, JsonSerial /// /// Saves the given Python dictionary, list, tuple or .NET object in JSON format for the specified path, - /// tolerating nested types the standard json module rejects: datetime/date/time are stored in ISO-8601 - /// format, Decimal and numpy values as numbers, NaN and infinity as null, .NET objects (Symbol included) - /// as the C# overload writes them and non-string dictionary keys as strings. - /// pandas objects are rejected, use , and so is any other type that would - /// not round trip. A top-level Python str, int, float or bool is not accepted, use - /// for text + /// tolerating nested types the standard json module rejects: datetime/date/time, pandas Timestamp and + /// numpy datetime64 are stored in ISO-8601 format, timedelta and timedelta64 as ISO-8601 durations, + /// Decimal and numpy numbers as numbers, NaN, infinity and NaT as null, .NET objects (Symbol included) + /// as the C# overload writes them (a NaN inside one is Newtonsoft's "NaN" + /// string) and non-string dictionary keys as strings. pandas DataFrames and Series are rejected, use + /// , and so is any other type that would not round trip. A top-level Python + /// str, int, float or bool is not accepted, use for text /// /// The object path /// The Python object to be saved @@ -548,7 +549,7 @@ private static void EnsurePythonHelpers() if (_jsonSerializeMethod == null) { var module = PyModule.FromString("object_store_helpers", @"from json import dumps, loads -from datetime import datetime, date, time +from datetime import datetime, date, time, timedelta from decimal import Decimal from math import isfinite import numpy @@ -558,16 +559,15 @@ import Newtonsoft.Json as newtonsoft def _key(key): # json only accepts str/int/float/bool/None keys; a .NET key is its string, like a .NET dictionary key + if isinstance(key, float) and not isfinite(key): + raise TypeError(f'save_json() cannot use {key} as a dictionary key') if isinstance(key, (str, int, float)) or key is None: return key - if isinstance(key, (datetime, date, time)): - return key.isoformat() - if isinstance(key, Decimal): - return str(key) - if isinstance(key, numpy.generic): - return key.item() if isinstance(key, System.Object): return str(key) + normalized = _normalize(key) + if isinstance(normalized, (str, int, float)) and not isinstance(normalized, bool): + return normalized raise TypeError(f'save_json() cannot use a {type(key).__name__} as a dictionary key') def _normalize(value): @@ -581,11 +581,18 @@ def _normalize(value): # NaN and infinity are not valid JSON return value if isfinite(value) else None if isinstance(value, (datetime, date, time)): - return value.isoformat() + return None if value is pandas.NaT else value.isoformat() + if isinstance(value, timedelta): + return pandas.Timedelta(value).isoformat() if isinstance(value, Decimal): return _normalize(float(value)) if isinstance(value, numpy.ndarray): - return _normalize(value.tolist()) + # tolist() turns nanosecond datetime64/timedelta64 into integers: convert those element-wise + return [_normalize(v) for v in value] if value.dtype.kind in 'mM' else _normalize(value.tolist()) + if isinstance(value, numpy.datetime64): + return None if numpy.isnat(value) else pandas.Timestamp(value).isoformat() + if isinstance(value, numpy.timedelta64): + return None if numpy.isnat(value) else pandas.Timedelta(value).isoformat() if isinstance(value, numpy.generic): return _normalize(value.item()) if isinstance(value, (pandas.DataFrame, pandas.Series)): diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index edf67758fd27..6f0cb8f01ec6 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -1257,6 +1257,83 @@ def test(object_store): "); } + [Test] + public void PythonSaveJsonWritesNumpyDatetimesAsIso8601AndNatAsNull() + { + RunPythonTest(@" +import numpy as np + +def test(object_store): + object_store.save_json('ergonomics/datetime64.json', { + 'index': pd.DatetimeIndex([pd.Timestamp('2020-01-01'), pd.Timestamp('2020-01-02 03:04:05.000000006')]).values, + 'scalar': np.datetime64('2020-01-01T00:00:00.000000000'), + 'day': np.datetime64('2020-01-01'), + 'timestamp': pd.Timestamp('2020-01-01 10:00'), + 'nat': pd.NaT, + 'nat64': np.datetime64('NaT'), + 'with_nat': pd.to_datetime(['2020-01-01', None]).values, + 'grid': np.array([['2020-01-01', '2020-01-02']], dtype='datetime64[D]') + }) + data = object_store.read_json('ergonomics/datetime64.json') + assert data == { + 'index': ['2020-01-01T00:00:00', '2020-01-02T03:04:05.000000006'], + 'scalar': '2020-01-01T00:00:00', + 'day': '2020-01-01T00:00:00', + 'timestamp': '2020-01-01T10:00:00', + 'nat': None, + 'nat64': None, + 'with_nat': ['2020-01-01T00:00:00', None], + 'grid': [['2020-01-01T00:00:00', '2020-01-02T00:00:00']] + }, data + + object_store.delete('ergonomics/datetime64.json') +"); + } + + [Test] + public void PythonSaveJsonWritesTimedeltasAsIso8601Durations() + { + RunPythonTest(@" +import numpy as np +from datetime import timedelta + +def test(object_store): + object_store.save_json('ergonomics/timedelta.json', { + 'holding': timedelta(days=1, hours=2), + 'latency': np.timedelta64(1500, 'ms'), + 'spans': (pd.to_datetime(['2020-01-02', '2020-01-03']) - pd.Timestamp('2020-01-01')).values, + 'pandas': pd.Timedelta(seconds=1.5), + 'nat': np.timedelta64('NaT') + }) + data = object_store.read_json('ergonomics/timedelta.json') + assert data == { + 'holding': 'P1DT2H0M0S', + 'latency': 'P0DT0H0M1.5S', + 'spans': ['P1DT0H0M0S', 'P2DT0H0M0S'], + 'pandas': 'P0DT0H0M1.5S', + 'nat': None + }, data + assert pd.Timedelta(data['holding']) == timedelta(days=1, hours=2) + + object_store.delete('ergonomics/timedelta.json') +"); + } + + [Test] + public void PythonSaveJsonRejectsNanDictionaryKeysWithTypeError() + { + RunPythonTest(@" +def test(object_store): + error = None + try: + object_store.save_json('ergonomics/nan_key.json', {float('nan'): 1}) + except TypeError as e: + error = str(e) + assert error is not None and 'nan' in error and 'key' in error, error + assert not object_store.contains_key('ergonomics/nan_key.json') +"); + } + [Test] public void PythonSaveJsonRejectsPandasObjectsPointingToSaveDataframe() { From 980a4ea692b7bbf042b2ee50ca8bee0882fc3eef Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 2 Nov 2026 13:31:06 -0400 Subject: [PATCH 06/14] Expose ObjectStore.GetKeyIssues as the single source of key diagnostics - GetKeyIssues returns null for a supported key, else the issues, the key rules and the SanitizeKey hint, so tools can validate keys ahead of time with the same rules - ValidateKey builds its exception from it --- Common/Storage/ObjectStore.cs | 27 +++++++++++++++---- Tests/Common/Storage/LocalObjectStoreTests.cs | 6 ++++- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index ac8dcd15ee85..0f02d819fc5f 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -210,17 +210,34 @@ public static bool IsSupportedKey(string key) /// The key is not supported. The message states the key rules public static void ValidateKey(string key) { - if (!IsSupportedKey(key)) + var issues = GetKeyIssues(key); + if (issues != null) { - throw new ArgumentException($"ObjectStore: path is not supported: '{key}': {string.Join("; ", GetKeyIssues(key))}." + - $" Object store {SupportedKeyRules}. Use ObjectStore.{nameof(SanitizeKey)}(name) to convert an arbitrary name into a supported path."); + throw new ArgumentException($"ObjectStore: path is not supported: '{key}': {issues}"); } } /// - /// Describes why a key fails , one entry per problem found + /// Describes why a key fails : the problems found, the key rules and the + /// hint, as stated in the error. Null for a supported key. + /// The single source for tools that validate keys ahead of time /// - private static List GetKeyIssues(string key) + /// The object store key to describe + /// The issues with the key, or null if it is supported + public static string GetKeyIssues(string key) + { + if (IsSupportedKey(key)) + { + return null; + } + return $"{string.Join("; ", FindKeyIssues(key))}. Object store {SupportedKeyRules}." + + $" Use ObjectStore.{nameof(SanitizeKey)}(name) to convert an arbitrary name into a supported path."; + } + + /// + /// Lists why a key fails , one entry per problem found + /// + private static List FindKeyIssues(string key) { var issues = new List(); if (string.IsNullOrEmpty(key)) diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index 6f0cb8f01ec6..2fa970fa5be5 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -595,10 +595,14 @@ public void ValidatesKeys(string key, bool isSupported) if (isSupported) { Assert.DoesNotThrow(() => ObjectStore.ValidateKey(key)); + Assert.IsNull(ObjectStore.GetKeyIssues(key)); } else { - Assert.Throws(() => ObjectStore.ValidateKey(key)); + var error = Assert.Throws(() => ObjectStore.ValidateKey(key)); + var issues = ObjectStore.GetKeyIssues(key); + Assert.IsNotNull(issues); + Assert.IsTrue(error.Message.EndsWith(issues), error.Message); } } From f715c84448987f8ae2b0669bdc467ec2760ac9a0 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 3 Nov 2026 13:54:49 -0400 Subject: [PATCH 07/14] Sanitize unsupported keys with a single warning and move the Python helpers to PythonObjectStore --- ...bjectStoreErgonomicsRegressionAlgorithm.cs | 57 +- ...bjectStoreErgonomicsRegressionAlgorithm.py | 47 +- Algorithm/QCAlgorithm.cs | 3 +- .../Python/Wrappers/AlgorithmPythonWrapper.cs | 2 +- Common/Storage/ObjectStore.cs | 502 ++++++------------ Common/Storage/PythonObjectStore.cs | 223 ++++++++ Research/QuantBook.cs | 6 +- Tests/Common/Storage/LocalObjectStoreTests.cs | 215 +++++--- Tests/Python/AlgorithmPythonWrapperTests.cs | 27 + 9 files changed, 588 insertions(+), 494 deletions(-) create mode 100644 Common/Storage/PythonObjectStore.cs diff --git a/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs b/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs index 1c7235fd7dd0..a290e329de2a 100644 --- a/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs +++ b/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs @@ -20,9 +20,8 @@ namespace QuantConnect.Algorithm.CSharp { /// - /// Regression algorithm asserting the ObjectStore ergonomics: unsupported keys are rejected with the - /// key rules stated in the error, converts arbitrary names - /// into supported keys, and + /// Regression algorithm asserting the ObjectStore ergonomics: an unsupported key is sanitized and used on + /// every save and read, and /// are aliases of Save and Read, and reading a missing key lists the available keys in the error. The Python /// version of this algorithm additionally covers the tolerant save_json/read_json and save_dataframe/read_dataframe helpers /// @@ -46,46 +45,20 @@ public override void Initialize() /// public override void OnEndOfAlgorithm() { - // an unsupported key, e.g. built from a user-facing name, is rejected stating the rules and the fix + // an unsupported key, e.g. built from a user-facing name, is sanitized and used on every save and read const string invalidKey = "trade_log_ai_hardware_&_cloud.csv"; - var errorMessage = string.Empty; - try - { - ObjectStore.Save(invalidKey, "a,b\n1,2"); - } - catch (ArgumentException exception) - { - errorMessage = exception.Message; - } - if (!errorMessage.Contains("unsupported character '&'") || !errorMessage.Contains("keys may only contain") || - !errorMessage.Contains("SanitizeKey")) + const string sanitizedKey = "trade_log_ai_hardware___cloud.csv"; + if (!ObjectStore.Save(invalidKey, "a,b\n1,2")) { - throw new RegressionTestException($"Expected the key issue and rules in the unsupported key error, got: '{errorMessage}'"); + throw new RegressionTestException("Expected the unsupported key to be sanitized and saved"); } - // ValidateKey states the same error eagerly, when the key is built - errorMessage = string.Empty; - try + if (!ObjectStore.Keys.Contains(sanitizedKey) || ObjectStore.Keys.Contains(invalidKey)) { - Storage.ObjectStore.ValidateKey(invalidKey); + throw new RegressionTestException($"Expected the object to be stored under '{sanitizedKey}', keys: {string.Join(", ", ObjectStore.Keys)}"); } - catch (ArgumentException exception) + if (!ObjectStore.ContainsKey(invalidKey) || ObjectStore.Read(invalidKey) != "a,b\n1,2" || ObjectStore.Read(sanitizedKey) != "a,b\n1,2") { - errorMessage = exception.Message; - } - if (!errorMessage.Contains("unsupported character '&'")) - { - throw new RegressionTestException($"Expected ValidateKey to reject the key stating the issue, got: '{errorMessage}'"); - } - - // SanitizeKey converts the arbitrary name into a supported key - var sanitized = Storage.ObjectStore.SanitizeKey(invalidKey); - if (sanitized != "trade_log_ai_hardware___cloud.csv") - { - throw new RegressionTestException($"Unexpected sanitized key: '{sanitized}'"); - } - if (!Storage.ObjectStore.IsSupportedKey(sanitized) || !ObjectStore.Save(sanitized, "a,b\n1,2")) - { - throw new RegressionTestException("Expected the sanitized key to be storable"); + throw new RegressionTestException("Expected the object to be read through both the original and the sanitized key"); } // SaveText and ReadText are aliases of Save and Read @@ -97,7 +70,7 @@ public override void OnEndOfAlgorithm() } // reading a missing key lists the available keys in the error - errorMessage = string.Empty; + var errorMessage = string.Empty; try { ObjectStore.ReadBytes("ergonomics_missing.json"); @@ -106,13 +79,17 @@ public override void OnEndOfAlgorithm() { errorMessage = exception.Message; } - if (!errorMessage.Contains("Keys: [") || !errorMessage.Contains($"'{sanitized}'")) + if (!errorMessage.Contains("Keys: [") || !errorMessage.Contains($"'{sanitizedKey}'")) { throw new RegressionTestException($"Expected the available keys in the missing key error, got: '{errorMessage}'"); } - ObjectStore.Delete(sanitized); + ObjectStore.Delete(invalidKey); ObjectStore.Delete("ergonomics_report.txt"); + if (ObjectStore.ContainsKey(sanitizedKey)) + { + throw new RegressionTestException("Expected the delete through the original key to remove the sanitized one"); + } } /// diff --git a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py index b7d8ef6e9aeb..a0ffad2e485a 100644 --- a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py +++ b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py @@ -16,9 +16,9 @@ from decimal import Decimal ### -### Regression algorithm asserting the ObjectStore ergonomics: unsupported keys are rejected with the -### key rules stated in the error, sanitize_key converts arbitrary names into supported keys, save_text and -### read_text are aliases of save and read, reading a missing key lists the available keys in the error, and +### Regression algorithm asserting the ObjectStore ergonomics: an unsupported key is sanitized and used on +### every save and read, save_text and read_text are aliases of save and read, reading a missing key lists the +### available keys in the error, and ### the tolerant save_json/read_json and save_dataframe/read_dataframe helpers handle datetime/date/Decimal/Symbol ### and data frames out of the box. ### @@ -33,32 +33,17 @@ def initialize(self): self.spy = self.add_equity("SPY", Resolution.DAILY).symbol def on_end_of_algorithm(self): - # an unsupported key, e.g. built from a user-facing name, is rejected stating the rules and the fix. - # note: BaseException because AlgorithmImports shadows the builtin Exception with System.Exception + # an unsupported key, e.g. built from a user-facing name, is sanitized and used on every save and read invalid_key = "trade_log_ai_hardware_&_cloud.csv" - error = None - try: - self.object_store.save(invalid_key, "a,b\n1,2") - except BaseException as e: - error = str(e) - if error is None or "unsupported character '&'" not in error or "keys may only contain" not in error or "SanitizeKey" not in error: - raise AssertionError(f"Expected the key issue and rules in the unsupported key error, got: '{error}'") - - # validate_key states the same error eagerly, when the key is built - error = None - try: - self.object_store.validate_key(invalid_key) - except BaseException as e: - error = str(e) - if error is None or "unsupported character '&'" not in error: - raise AssertionError(f"Expected validate_key to reject the key stating the issue, got: '{error}'") - - # sanitize_key converts the arbitrary name into a supported key - sanitized = self.object_store.sanitize_key(invalid_key) - if sanitized != "trade_log_ai_hardware___cloud.csv": - raise AssertionError(f"Unexpected sanitized key: '{sanitized}'") - if not self.object_store.save(sanitized, "a,b\n1,2"): - raise AssertionError("Expected the sanitized key to be storable") + sanitized_key = "trade_log_ai_hardware___cloud.csv" + if not self.object_store.save(invalid_key, "a,b\n1,2"): + raise AssertionError("Expected the unsupported key to be sanitized and saved") + keys = list(self.object_store.keys) + if sanitized_key not in keys or invalid_key in keys: + raise AssertionError(f"Expected the object to be stored under '{sanitized_key}', keys: {keys}") + if not self.object_store.contains_key(invalid_key) or self.object_store.read(invalid_key) != "a,b\n1,2" \ + or self.object_store.read(sanitized_key) != "a,b\n1,2": + raise AssertionError("Expected the object to be read through both the original and the sanitized key") # save_json tolerates datetime/date/Decimal/numpy/NaN values, .NET objects and non-string dictionary keys trade_log = { @@ -131,8 +116,10 @@ def on_end_of_algorithm(self): self.object_store.read_bytes("ergonomics/missing.json") except BaseException as e: error = str(e) - if error is None or "Keys: [" not in error or f"'{sanitized}'" not in error: + if error is None or "Keys: [" not in error or f"'{sanitized_key}'" not in error: raise AssertionError(f"Expected the available keys in the missing key error, got: '{error}'") - for key in [sanitized, "ergonomics/trade_log.json", "ergonomics/history.csv", "ergonomics/report.txt"]: + for key in [invalid_key, "ergonomics/trade_log.json", "ergonomics/history.csv", "ergonomics/report.txt"]: self.object_store.delete(key) + if self.object_store.contains_key(sanitized_key): + raise AssertionError("Expected the delete through the original key to remove the sanitized one") diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index 5cefc6375a19..315d70a21af1 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -3308,7 +3308,8 @@ public void SetApi(IApi api) [DocumentationAttribute(MachineLearning)] public void SetObjectStore(IObjectStore objectStore) { - ObjectStore = new ObjectStore(objectStore); + // an already wrapped store, like the Python one, is used as is + ObjectStore = objectStore as ObjectStore ?? new ObjectStore(objectStore, Debug); } /// diff --git a/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs b/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs index 473aa82cca9b..526906505687 100644 --- a/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs +++ b/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs @@ -1231,7 +1231,7 @@ public void SetCurrentSlice(Slice slice) /// Sets the object store /// /// The object store - public void SetObjectStore(IObjectStore objectStore) => _baseAlgorithm.SetObjectStore(objectStore); + public void SetObjectStore(IObjectStore objectStore) => _baseAlgorithm.SetObjectStore(new PythonObjectStore(objectStore, _baseAlgorithm.Debug)); /// /// Determines if the Symbol is shortable at the brokerage diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index 0f02d819fc5f..da90b5fd529b 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -15,21 +15,26 @@ using System; using System.Collections; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; +using System.Threading; using System.Text.RegularExpressions; using System.Xml.Serialization; using Newtonsoft.Json; -using Python.Runtime; using QuantConnect.Interfaces; +using QuantConnect.Logging; using QuantConnect.Packets; namespace QuantConnect.Storage { /// - /// Helper class for easier access to methods + /// Helper class for easier access to methods. A key that does not follow the key + /// format (english letters, numbers, spaces and the characters '/', '\', '_', '#', '-', '$' and '=', plus at + /// most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv') is + /// replaced by its sanitized version on every read and save, with a single warning the first time it happens /// public class ObjectStore : IObjectStore { @@ -38,15 +43,11 @@ public class ObjectStore : IObjectStore private static readonly Regex UnsupportedKeyCharactersRegex = new(@"[^a-zA-Z0-9\\/_#\-\$= .]", RegexOptions.Compiled); private static readonly Regex KeyExtensionRegex = new(@"^[a-zA-Z0-9]+$", RegexOptions.Compiled); - private const string SupportedKeyRules = "keys may only contain english letters, numbers, spaces and the characters" + - " '/', '\\', '_', '#', '-', '$' and '=', plus at most one '.' followed by a letters-and-numbers-only extension," + - " e.g. 'folder/trade_log-2024.csv'"; - - // Python helpers for the PyObject APIs, lazily created under the GIL - private static PyObject _jsonSerializeMethod; - private static PyObject _jsonDeserializeMethod; - private static PyObject _dataFrameSerializeMethod; - private static PyObject _dataFrameDeserializeMethod; + private readonly IObjectStore _store; + private readonly Action _warn; + private int _warned; + // every key used so far, mapped to the key the store is asked for, so each one is checked once + private readonly ConcurrentDictionary _keys = new(); /// /// Gets the maximum storage limit in bytes @@ -67,15 +68,16 @@ public event EventHandler ErrorRaised remove { _store.ErrorRaised -= value; } } - private readonly IObjectStore _store; - /// /// Initializes a new instance of the class /// /// The instance to wrap - public ObjectStore(IObjectStore store) + /// Receives the single warning logged the first time an unsupported key is sanitized, + /// the engine log by default + public ObjectStore(IObjectStore store, Action warn = null) { _store = store; + _warn = warn ?? (message => Log.Trace(message)); } /// @@ -108,7 +110,7 @@ public void Initialize(int userId, int projectId, string userToken, Controls con /// True if the key was found public bool ContainsKey(string path) { - return _store.ContainsKey(path); + return _store.ContainsKey(GetKey(path)); } /// @@ -118,18 +120,18 @@ public bool ContainsKey(string path) /// A byte array containing the data public byte[] ReadBytes(string path) { - return _store.ReadBytes(path); + return _store.ReadBytes(GetKey(path)); } /// /// Saves the object data for the specified path /// - /// The object path + /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' /// The object data /// True if the save operation was successful public bool SaveBytes(string path, byte[] contents) { - return _store.SaveBytes(path, contents); + return _store.SaveBytes(GetKey(path), contents); } /// @@ -139,7 +141,7 @@ public bool SaveBytes(string path, byte[] contents) /// True if the delete operation was successful public bool Delete(string path) { - return _store.Delete(path); + return _store.Delete(GetKey(path)); } /// @@ -149,7 +151,7 @@ public bool Delete(string path) /// The path for the file public string GetFilePath(string path) { - return _store.GetFilePath(path); + return _store.GetFilePath(GetKey(path)); } /// @@ -162,7 +164,7 @@ public string Read(string path, Encoding encoding = null) { encoding = encoding ?? Encoding.UTF8; - var data = _store.ReadBytes(path); + var data = ReadBytes(path); return data != null ? encoding.GetString(data) : null; } @@ -188,141 +190,6 @@ public string ReadText(string path, Encoding encoding = null) return Read(path, encoding); } - /// - /// Determines whether the given key follows the object store key format: english letters, numbers, spaces - /// and the characters '/', '\', '_', '#', '-', '$' and '=', plus at most one '.' followed by a - /// letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv' - /// - /// The object store key to validate - /// True if the key is supported - public static bool IsSupportedKey(string key) - { - return !string.IsNullOrEmpty(key) && SupportedKeyRegex.IsMatch(key) - // just in case - && key.Count(c => c == '/') <= 100 && key.Count(c => c == '\\') <= 100; - } - - /// - /// Throws if the given key does not follow the object store key format, see . - /// Use it to validate programmatically-built keys when they are constructed, instead of at the final save - /// - /// The object store key to validate - /// The key is not supported. The message states the key rules - public static void ValidateKey(string key) - { - var issues = GetKeyIssues(key); - if (issues != null) - { - throw new ArgumentException($"ObjectStore: path is not supported: '{key}': {issues}"); - } - } - - /// - /// Describes why a key fails : the problems found, the key rules and the - /// hint, as stated in the error. Null for a supported key. - /// The single source for tools that validate keys ahead of time - /// - /// The object store key to describe - /// The issues with the key, or null if it is supported - public static string GetKeyIssues(string key) - { - if (IsSupportedKey(key)) - { - return null; - } - return $"{string.Join("; ", FindKeyIssues(key))}. Object store {SupportedKeyRules}." + - $" Use ObjectStore.{nameof(SanitizeKey)}(name) to convert an arbitrary name into a supported path."; - } - - /// - /// Lists why a key fails , one entry per problem found - /// - private static List FindKeyIssues(string key) - { - var issues = new List(); - if (string.IsNullOrEmpty(key)) - { - issues.Add("the key is empty"); - return issues; - } - - var unsupported = UnsupportedKeyCharactersRegex.Matches(key).Select(match => $"'{match.Value}'").Distinct().ToList(); - if (unsupported.Count > 0) - { - issues.Add($"unsupported character{(unsupported.Count > 1 ? "s" : "")} {string.Join(", ", unsupported)}"); - } - - // a single leading '.' is allowed, every other '.' must introduce the final extension - var name = key.TrimStart('.'); - var dots = name.Count(c => c == '.'); - if (dots > 1) - { - issues.Add($"it has {dots} '.' but at most one '.extension' is allowed"); - } - else if (dots == 1) - { - var extension = name.Substring(name.IndexOf('.') + 1); - if (extension.Length > 0 && !KeyExtensionRegex.IsMatch(extension)) - { - issues.Add($"the extension '{extension}' may only contain letters and numbers"); - } - } - if (name.Length == 0 || name[0] == '.') - { - issues.Add("it has no name before the '.'"); - } - - if (key.Count(c => c == '/') > 100 || key.Count(c => c == '\\') > 100) - { - issues.Add("it has more than 100 '/' or '\\' separators"); - } - - if (issues.Count == 0) - { - issues.Add("it does not follow the key format"); - } - return issues; - } - - /// - /// Converts an arbitrary name into a supported object store key by replacing every unsupported character - /// with '_', keeping at most the final '.extension'. Useful for programmatically-built keys, for instance - /// from user-facing names: 'trade log: AI & Cloud.csv' becomes 'trade log_ AI _ Cloud.csv'. - /// The mapping is lossy: distinct names can sanitize to the same key, e.g. 'a&b.csv' and 'a:b.csv' both - /// become 'a_b.csv', so it only sanitizes, it does not generate unique keys. Checking for collisions, - /// for instance with , is up to the caller - /// - /// The arbitrary name to sanitize - /// A key following the object store key format, see - public static string SanitizeKey(string key) - { - if (string.IsNullOrEmpty(key)) - { - throw new ArgumentException("ObjectStore.SanitizeKey(): key cannot be null or empty", nameof(key)); - } - if (IsSupportedKey(key)) - { - return key; - } - - var sanitized = UnsupportedKeyCharactersRegex.Replace(key, "_"); - - // a single trailing '.extension' of letters and numbers is allowed: keep the last dot if it - // introduces a valid extension and replace every other dot - var lastDot = sanitized.LastIndexOf('.'); - if (lastDot > 0 && KeyExtensionRegex.IsMatch(sanitized.Substring(lastDot + 1))) - { - sanitized = sanitized.Substring(0, lastDot).Replace('.', '_') + sanitized.Substring(lastDot); - } - else if (lastDot >= 0) - { - sanitized = sanitized.Replace('.', '_'); - } - - ValidateKey(sanitized); - return sanitized; - } - /// /// Returns the JSON deserialized object data for the specified path /// @@ -338,42 +205,6 @@ public T ReadJson(string path, Encoding encoding = null, JsonSerializerSettin return JsonConvert.DeserializeObject(json, settings); } - /// - /// Returns the JSON deserialized object data for the specified path as Python objects - /// - /// The object path - /// The deserialized Python object - public PyObject ReadJson(string path) - { - var json = Read(path); - using (Py.GIL()) - { - EnsurePythonHelpers(); - using var pyJson = json.ToPython(); - return _jsonDeserializeMethod.Invoke(pyJson); - } - } - - /// - /// Returns the object data for the specified path as a pandas DataFrame, read as JSON if the path has - /// a '.json' extension and as CSV otherwise, so a round trip restores the frame. - /// A CSV index in ISO-8601 format is restored as a time index. A saved Series comes back as a - /// one-column DataFrame, call squeeze() on it to get the Series - /// - /// The object path - /// The pandas DataFrame - public PyObject ReadDataframe(string path) - { - var serialized = Read(path); - using (Py.GIL()) - { - EnsurePythonHelpers(); - using var pySerialized = serialized.ToPython(); - using var pyPath = path.ToPython(); - return _dataFrameDeserializeMethod.Invoke(pySerialized, pyPath); - } - } - /// /// Returns the XML deserialized object data for the specified path /// @@ -397,7 +228,7 @@ public T ReadXml(string path, Encoding encoding = null) /// Saves the data from a local file path associated with the specified path /// /// If the file does not exist it will throw an exception - /// The object path + /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' /// True if the object was saved successfully public bool Save(string path) { @@ -409,41 +240,39 @@ public bool Save(string path) } var bytes = File.ReadAllBytes(filePath); - return _store.SaveBytes(path, bytes); + return SaveBytes(path, bytes); } /// /// Saves the object data in text format for the specified path /// - /// The object path + /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' /// The string object to be saved /// The string encoding used, by default /// True if the object was saved successfully public bool Save(string path, string text, Encoding encoding = null) { encoding ??= Encoding.UTF8; - return _store.SaveBytes(path, encoding.GetBytes(text)); + return SaveBytes(path, encoding.GetBytes(text)); } /// /// Saves the object data in text format for the specified path /// - /// The object path + /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' /// The string object to be saved /// The string encoding used /// True if the object was saved successfully public bool SaveString(string path, string text, Encoding encoding = null) { - encoding = encoding ?? Encoding.UTF8; - - return _store.SaveBytes(path, encoding.GetBytes(text)); + return Save(path, text, encoding); } /// /// Saves the object data in text format for the specified path. /// Alias of /// - /// The object path + /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' /// The string object to be saved /// The string encoding used, by default /// True if the object was saved successfully @@ -455,7 +284,7 @@ public bool SaveText(string path, string text, Encoding encoding = null) /// /// Saves the object data in JSON format for the specified path /// - /// The object path + /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' /// The object to be saved /// The string encoding used /// The settings used by the JSON serializer @@ -468,56 +297,10 @@ public bool SaveJson(string path, T obj, Encoding encoding = null, JsonSerial return SaveString(path, json, encoding); } - /// - /// Saves the given Python dictionary, list, tuple or .NET object in JSON format for the specified path, - /// tolerating nested types the standard json module rejects: datetime/date/time, pandas Timestamp and - /// numpy datetime64 are stored in ISO-8601 format, timedelta and timedelta64 as ISO-8601 durations, - /// Decimal and numpy numbers as numbers, NaN, infinity and NaT as null, .NET objects (Symbol included) - /// as the C# overload writes them (a NaN inside one is Newtonsoft's "NaN" - /// string) and non-string dictionary keys as strings. pandas DataFrames and Series are rejected, use - /// , and so is any other type that would not round trip. A top-level Python - /// str, int, float or bool is not accepted, use for text - /// - /// The object path - /// The Python object to be saved - /// True if the object was saved successfully - /// A TypeError naming the value that cannot be serialized - public bool SaveJson(string path, PyObject obj) - { - string json; - using (Py.GIL()) - { - EnsurePythonHelpers(); - using var result = _jsonSerializeMethod.Invoke(obj); - json = result.As(); - } - return Save(path, json); - } - - /// - /// Saves the given pandas DataFrame (or Series) for the specified path, in JSON format if the - /// path has a '.json' extension and as CSV otherwise - /// - /// The object path - /// The pandas DataFrame or Series to be saved - /// True if the object was saved successfully - public bool SaveDataframe(string path, PyObject dataFrame) - { - string serialized; - using (Py.GIL()) - { - EnsurePythonHelpers(); - using var pyPath = (path ?? string.Empty).ToPython(); - using var result = _dataFrameSerializeMethod.Invoke(dataFrame, pyPath); - serialized = result.As(); - } - return Save(path, serialized); - } - /// /// Saves the object data in XML format for the specified path /// - /// The object path + /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' /// The object to be saved /// The string encoding used /// True if the object was saved successfully @@ -559,99 +342,140 @@ public void Dispose() } /// - /// Lazily creates the Python helper methods backing the PyObject APIs. Must be called under the GIL + /// Throws if the given key does not follow the key format, stating every problem found and the rules. + /// The last line of defense of the store implementations, as this class only hands them supported keys + /// + /// The object store key to validate + /// The key is not supported + internal static void ValidateKey(string key) + { + var issues = GetKeyIssues(key); + if (issues != null) + { + throw new ArgumentException($"ObjectStore: path is not supported: '{key}': {issues}"); + } + } + + /// + /// The key the store is asked for: the given one when it follows the key format, else its sanitized + /// version, warning only the first time any key is sanitized. Null and empty keys are left for the store to reject + /// + private string GetKey(string path) + { + if (string.IsNullOrEmpty(path)) + { + return path; + } + if (_keys.TryGetValue(path, out var key)) + { + return key; + } + + var issues = GetKeyIssues(path); + key = issues == null ? path : SanitizeKey(path); + if (_keys.TryAdd(path, key) && issues != null && Interlocked.Exchange(ref _warned, 1) == 0) + { + _warn($"ObjectStore: key '{path}' is not supported: {issues}. Using '{key}' instead, now and on every" + + " other read or save of this key. Any other unsupported key is sanitized the same way, without further warnings"); + } + return key; + } + + private static bool IsSupportedKey(string key) + { + return !string.IsNullOrEmpty(key) && SupportedKeyRegex.IsMatch(key) + // just in case + && key.Count(c => c == '/') <= 100 && key.Count(c => c == '\\') <= 100; + } + + /// + /// Describes why a key does not follow the key format: the problems found and the rules. Null for a supported key + /// + private static string GetKeyIssues(string key) + { + if (IsSupportedKey(key)) + { + return null; + } + return string.Join("; ", FindKeyIssues(key)) + + ". Keys may only contain english letters, numbers, spaces and the characters" + + " '/', '\\', '_', '#', '-', '$' and '=', plus at most one '.' followed by a letters-and-numbers-only extension," + + " e.g. 'folder/trade_log-2024.csv'"; + } + + /// + /// Lists why a key fails , one entry per problem found /// - private static void EnsurePythonHelpers() + private static List FindKeyIssues(string key) { - if (_jsonSerializeMethod == null) + var issues = new List(); + if (string.IsNullOrEmpty(key)) + { + issues.Add("the key is empty"); + return issues; + } + + var unsupported = UnsupportedKeyCharactersRegex.Matches(key).Select(match => $"'{match.Value}'").Distinct().ToList(); + if (unsupported.Count > 0) + { + issues.Add($"unsupported character{(unsupported.Count > 1 ? "s" : "")} {string.Join(", ", unsupported)}"); + } + + // a single leading '.' is allowed, every other '.' must introduce the final extension + var name = key.TrimStart('.'); + var dots = name.Count(c => c == '.'); + if (dots > 1) + { + issues.Add($"it has {dots} '.' but at most one '.extension' is allowed"); + } + else if (dots == 1) { - var module = PyModule.FromString("object_store_helpers", @"from json import dumps, loads -from datetime import datetime, date, time, timedelta -from decimal import Decimal -from math import isfinite -import numpy -import pandas -import System -import Newtonsoft.Json as newtonsoft - -def _key(key): - # json only accepts str/int/float/bool/None keys; a .NET key is its string, like a .NET dictionary key - if isinstance(key, float) and not isfinite(key): - raise TypeError(f'save_json() cannot use {key} as a dictionary key') - if isinstance(key, (str, int, float)) or key is None: - return key - if isinstance(key, System.Object): - return str(key) - normalized = _normalize(key) - if isinstance(normalized, (str, int, float)) and not isinstance(normalized, bool): - return normalized - raise TypeError(f'save_json() cannot use a {type(key).__name__} as a dictionary key') - -def _normalize(value): - if isinstance(value, dict): - return {_key(k): _normalize(v) for k, v in value.items()} - if isinstance(value, (list, tuple, set, frozenset)): - return [_normalize(v) for v in value] - if isinstance(value, (str, int)) or value is None: - return value - if isinstance(value, float): - # NaN and infinity are not valid JSON - return value if isfinite(value) else None - if isinstance(value, (datetime, date, time)): - return None if value is pandas.NaT else value.isoformat() - if isinstance(value, timedelta): - return pandas.Timedelta(value).isoformat() - if isinstance(value, Decimal): - return _normalize(float(value)) - if isinstance(value, numpy.ndarray): - # tolist() turns nanosecond datetime64/timedelta64 into integers: convert those element-wise - return [_normalize(v) for v in value] if value.dtype.kind in 'mM' else _normalize(value.tolist()) - if isinstance(value, numpy.datetime64): - return None if numpy.isnat(value) else pandas.Timestamp(value).isoformat() - if isinstance(value, numpy.timedelta64): - return None if numpy.isnat(value) else pandas.Timedelta(value).isoformat() - if isinstance(value, numpy.generic): - return _normalize(value.item()) - if isinstance(value, (pandas.DataFrame, pandas.Series)): - raise TypeError(f'save_json() does not accept a pandas {type(value).__name__}, use save_dataframe() instead') - if isinstance(value, System.Object): - # .NET objects serialize like the C# SaveJson does - return loads(newtonsoft.JsonConvert.SerializeObject(value)) - raise TypeError(f'save_json() cannot serialize a {type(value).__name__}: convert it to a dict, list or str first') - -def serialize(value): - return dumps(_normalize(value), allow_nan=False) - -def deserialize(json_string): - return loads(json_string) - -def serialize_dataframe(value, key): - if not hasattr(value, 'to_csv'): - raise TypeError(f'save_dataframe() expects a pandas DataFrame or Series but received {type(value).__name__}') - if key.lower().endswith('.json'): - return value.to_json() - return value.to_csv() - -def deserialize_dataframe(serialized, key): - import pandas - from io import StringIO - if key.lower().endswith('.json'): - return pandas.read_json(StringIO(serialized)) - # to_csv() writes the index as the first column, datetimes in ISO-8601: restore a time index when it parses as one - frame = pandas.read_csv(StringIO(serialized), index_col=0) - if pandas.api.types.is_string_dtype(frame.index): - try: - frame.index = pandas.to_datetime(frame.index, format='ISO8601') - except (ValueError, TypeError): - pass - return frame -"); - _jsonDeserializeMethod = module.GetAttr("deserialize"); - _dataFrameSerializeMethod = module.GetAttr("serialize_dataframe"); - _dataFrameDeserializeMethod = module.GetAttr("deserialize_dataframe"); - // last so partial initialization is never observed - _jsonSerializeMethod = module.GetAttr("serialize"); + var extension = name[(name.IndexOf('.', StringComparison.Ordinal) + 1)..]; + if (extension.Length > 0 && !KeyExtensionRegex.IsMatch(extension)) + { + issues.Add($"the extension '{extension}' may only contain letters and numbers"); + } } + if (name.Length == 0 || name[0] == '.') + { + issues.Add("it has no name before the '.'"); + } + + if (key.Count(c => c == '/') > 100 || key.Count(c => c == '\\') > 100) + { + issues.Add("it has more than 100 '/' or '\\' separators"); + } + + if (issues.Count == 0) + { + issues.Add("it does not follow the key format"); + } + return issues; + } + + /// + /// Converts an unsupported key into a supported one by replacing every unsupported character with '_', + /// keeping at most the final '.extension': 'trade log: AI & Cloud.csv' becomes 'trade log_ AI _ Cloud.csv'. + /// The mapping is lossy, distinct keys can sanitize to the same one. Throws when the key cannot be fixed + /// + private static string SanitizeKey(string key) + { + var sanitized = UnsupportedKeyCharactersRegex.Replace(key, "_"); + + // a single trailing '.extension' of letters and numbers is allowed: keep the last dot if it + // introduces a valid extension and replace every other dot + var lastDot = sanitized.LastIndexOf('.'); + if (lastDot > 0 && KeyExtensionRegex.IsMatch(sanitized.AsSpan(lastDot + 1))) + { + sanitized = string.Concat(sanitized[..lastDot].Replace('.', '_'), sanitized.AsSpan(lastDot)); + } + else if (lastDot >= 0) + { + sanitized = sanitized.Replace('.', '_'); + } + + ValidateKey(sanitized); + return sanitized; } } -} \ No newline at end of file +} diff --git a/Common/Storage/PythonObjectStore.cs b/Common/Storage/PythonObjectStore.cs new file mode 100644 index 000000000000..30c0a57ad8d3 --- /dev/null +++ b/Common/Storage/PythonObjectStore.cs @@ -0,0 +1,223 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using Python.Runtime; +using QuantConnect.Interfaces; + +namespace QuantConnect.Storage +{ + /// + /// The of Python algorithms and notebooks: adds the save and read methods that take + /// and return Python objects, such as dictionaries and pandas DataFrames + /// + public class PythonObjectStore : ObjectStore + { + // Python helpers for the PyObject APIs, lazily created under the GIL + private static PyObject _jsonSerializeMethod; + private static PyObject _jsonDeserializeMethod; + private static PyObject _dataFrameSerializeMethod; + private static PyObject _dataFrameDeserializeMethod; + + /// + /// Initializes a new instance of the class + /// + /// The instance to wrap + /// Receives the warning logged the first time an unsupported key is sanitized, + /// the engine log by default + public PythonObjectStore(IObjectStore store, Action warn = null) + : base(store, warn) + { + } + + /// + /// Returns the JSON deserialized object data for the specified path as Python objects + /// + /// The object path + /// The deserialized Python object + public PyObject ReadJson(string path) + { + var json = Read(path); + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var pyJson = json.ToPython(); + return _jsonDeserializeMethod.Invoke(pyJson); + } + } + + /// + /// Returns the object data for the specified path as a pandas DataFrame, read as JSON if the path has + /// a '.json' extension and as CSV otherwise, so a round trip restores the frame. + /// A CSV index in ISO-8601 format is restored as a time index. A saved Series comes back as a + /// one-column DataFrame, call squeeze() on it to get the Series + /// + /// The object path + /// The pandas DataFrame + public PyObject ReadDataframe(string path) + { + var serialized = Read(path); + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var pySerialized = serialized.ToPython(); + using var pyPath = path.ToPython(); + return _dataFrameDeserializeMethod.Invoke(pySerialized, pyPath); + } + } + + /// + /// Saves the given Python dictionary, list, tuple or .NET object in JSON format for the specified path, + /// tolerating nested types the standard json module rejects: datetime/date/time, pandas Timestamp and + /// numpy datetime64 are stored in ISO-8601 format, timedelta and timedelta64 as ISO-8601 durations, + /// Decimal and numpy numbers as numbers, NaN, infinity and NaT as null, .NET objects (Symbol included) + /// as the C# overload writes them (a NaN inside one is Newtonsoft's "NaN" + /// string) and non-string dictionary keys as strings. pandas DataFrames and Series are rejected, use + /// , and so is any other type that would not round trip. A top-level Python + /// str, int, float or bool is not accepted, use for text + /// + /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' + /// The Python object to be saved + /// True if the object was saved successfully + /// A TypeError naming the value that cannot be serialized + public bool SaveJson(string path, PyObject obj) + { + string json; + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var result = _jsonSerializeMethod.Invoke(obj); + json = result.As(); + } + return Save(path, json); + } + + /// + /// Saves the given pandas DataFrame (or Series) for the specified path, in JSON format if the + /// path has a '.json' extension and as CSV otherwise + /// + /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' + /// The pandas DataFrame or Series to be saved + /// True if the object was saved successfully + public bool SaveDataframe(string path, PyObject dataFrame) + { + string serialized; + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var pyPath = (path ?? string.Empty).ToPython(); + using var result = _dataFrameSerializeMethod.Invoke(dataFrame, pyPath); + serialized = result.As(); + } + return Save(path, serialized); + } + + /// + /// Lazily creates the Python helper methods backing the PyObject APIs. Must be called under the GIL + /// + private static void EnsurePythonHelpers() + { + if (_jsonSerializeMethod == null) + { + var module = PyModule.FromString("object_store_helpers", @"from json import dumps, loads +from datetime import datetime, date, time, timedelta +from decimal import Decimal +from math import isfinite +import numpy +import pandas +import System +import Newtonsoft.Json as newtonsoft + +def _key(key): + # json only accepts str/int/float/bool/None keys; a .NET key is its string, like a .NET dictionary key + if isinstance(key, float) and not isfinite(key): + raise TypeError(f'save_json() cannot use {key} as a dictionary key') + if isinstance(key, (str, int, float)) or key is None: + return key + if isinstance(key, System.Object): + return str(key) + normalized = _normalize(key) + if isinstance(normalized, (str, int, float)) and not isinstance(normalized, bool): + return normalized + raise TypeError(f'save_json() cannot use a {type(key).__name__} as a dictionary key') + +def _normalize(value): + if isinstance(value, dict): + return {_key(k): _normalize(v) for k, v in value.items()} + if isinstance(value, (list, tuple, set, frozenset)): + return [_normalize(v) for v in value] + if isinstance(value, (str, int)) or value is None: + return value + if isinstance(value, float): + # NaN and infinity are not valid JSON + return value if isfinite(value) else None + if isinstance(value, (datetime, date, time)): + return None if value is pandas.NaT else value.isoformat() + if isinstance(value, timedelta): + return pandas.Timedelta(value).isoformat() + if isinstance(value, Decimal): + return _normalize(float(value)) + if isinstance(value, numpy.ndarray): + # tolist() turns nanosecond datetime64/timedelta64 into integers: convert those element-wise + return [_normalize(v) for v in value] if value.dtype.kind in 'mM' else _normalize(value.tolist()) + if isinstance(value, numpy.datetime64): + return None if numpy.isnat(value) else pandas.Timestamp(value).isoformat() + if isinstance(value, numpy.timedelta64): + return None if numpy.isnat(value) else pandas.Timedelta(value).isoformat() + if isinstance(value, numpy.generic): + return _normalize(value.item()) + if isinstance(value, (pandas.DataFrame, pandas.Series)): + raise TypeError(f'save_json() does not accept a pandas {type(value).__name__}, use save_dataframe() instead') + if isinstance(value, System.Object): + # .NET objects serialize like the C# SaveJson does + return loads(newtonsoft.JsonConvert.SerializeObject(value)) + raise TypeError(f'save_json() cannot serialize a {type(value).__name__}: convert it to a dict, list or str first') + +def serialize(value): + return dumps(_normalize(value), allow_nan=False) + +def deserialize(json_string): + return loads(json_string) + +def serialize_dataframe(value, key): + if not hasattr(value, 'to_csv'): + raise TypeError(f'save_dataframe() expects a pandas DataFrame or Series but received {type(value).__name__}') + if key.lower().endswith('.json'): + return value.to_json() + return value.to_csv() + +def deserialize_dataframe(serialized, key): + import pandas + from io import StringIO + if key.lower().endswith('.json'): + return pandas.read_json(StringIO(serialized)) + # to_csv() writes the index as the first column, datetimes in ISO-8601: restore a time index when it parses as one + frame = pandas.read_csv(StringIO(serialized), index_col=0) + if pandas.api.types.is_string_dtype(frame.index): + try: + frame.index = pandas.to_datetime(frame.index, format='ISO8601') + except (ValueError, TypeError): + pass + return frame +"); + _jsonDeserializeMethod = module.GetAttr("deserialize"); + _dataFrameSerializeMethod = module.GetAttr("serialize_dataframe"); + _dataFrameDeserializeMethod = module.GetAttr("deserialize_dataframe"); + // last so partial initialization is never observed + _jsonSerializeMethod = module.GetAttr("serialize"); + } + } + } +} diff --git a/Research/QuantBook.cs b/Research/QuantBook.cs index 1840312d7d24..37b5a132d5e3 100644 --- a/Research/QuantBook.cs +++ b/Research/QuantBook.cs @@ -27,6 +27,7 @@ using QuantConnect.Securities.Future; using QuantConnect.Securities.Option; using QuantConnect.Statistics; +using QuantConnect.Storage; using QuantConnect.Util; using System; using System.Collections.Generic; @@ -165,7 +166,10 @@ public QuantBook() : base() StorageAccess = Config.GetValue("storage-permissions", new Packets.StoragePermissions()) }, AlgorithmMode.Research); - SetObjectStore(algorithmHandlers.ObjectStore); + // wrapped here so key warnings reach the console: a notebook has no debug message consumer + SetObjectStore(_isPythonNotebook + ? new PythonObjectStore(algorithmHandlers.ObjectStore) + : new ObjectStore(algorithmHandlers.ObjectStore)); _dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider); _dataProvider = algorithmHandlers.DataProvider; diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index 2fa970fa5be5..d5baca3df3bf 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -19,6 +19,7 @@ using NUnit.Framework; using System.Threading; using QuantConnect.Util; +using QuantConnect.Algorithm; using QuantConnect.Logging; using QuantConnect.Packets; using QuantConnect.Storage; @@ -36,7 +37,8 @@ public class LocalObjectStoreTests private static readonly string TestStorageRoot = $"{Directory.GetCurrentDirectory()}/{nameof(LocalObjectStoreTests)}"; private static readonly string StorageRootConfigurationValue = LocalObjectStore.DefaultObjectStore; - private ObjectStore _store; + // the Python store so the Python tests share it, the C# tests only reach the base members + private PythonObjectStore _store; private ILogHandler _logHandler; [OneTimeSetUp] @@ -44,7 +46,7 @@ public void Setup() { LocalObjectStore.DefaultObjectStore = TestStorageRoot; #pragma warning disable CA2000 - _store = new ObjectStore(new TestLocalObjectStore()); + _store = new PythonObjectStore(new TestLocalObjectStore()); #pragma warning restore CA2000 _store.Initialize(0, 0, "", new Controls() { StorageLimit = 5 * 1024 * 1024, StorageFileCount = 100 }, AlgorithmMode.Backtesting); @@ -231,9 +233,9 @@ public void DeletePermissions([Values] bool read, [Values] bool write, [Values] } } - [TestCase("../prefix/")] - [TestCase("..\\prefix/")] - public void InvalidCustomPathsStore(string path) + [TestCase("../prefix/", "__/prefix/")] + [TestCase("..\\prefix/", "__/prefix/")] + public void InvalidCustomPathsStore(string path, string sanitizedPath) { #pragma warning disable CA2000 using (var store = new ObjectStore(new TestLocalObjectStore())) @@ -242,7 +244,12 @@ public void InvalidCustomPathsStore(string path) store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); Assert.AreEqual(0, store.Count()); - Assert.Throws(() => store.SaveString($"{path}ILove", "Pizza")); + // '..' cannot leave the storage root: it is sanitized away + Assert.IsTrue(store.SaveString($"{path}ILove", "Pizza")); + Assert.AreEqual(new[] { $"{sanitizedPath}ILove" }, store.Keys); + Assert.That(Path.GetFullPath(store.GetFilePath($"{path}ILove")), Does.StartWith(Path.GetFullPath(TestStorageRoot))); + + store.Delete($"{path}ILove"); } } @@ -580,41 +587,54 @@ public void KeyNotFoundErrorTruncatesTheAvailableKeysListAtTen() } } - [TestCase("my_key", true)] - [TestCase("./my_key/nested\\file.csv", true)] - [TestCase("file with spaces #1-$=.txt", true)] - [TestCase("trade_log_ai_hardware_&_cloud.csv", false)] - [TestCase("file.tar.gz", false)] - [TestCase("file.name_v2", false)] - [TestCase("**abc**", false)] - [TestCase("", false)] - [TestCase(null, false)] - public void ValidatesKeys(string key, bool isSupported) + [TestCase("my_key/file.csv", "my_key/file.csv")] + [TestCase("trade_log_ai_hardware_&_cloud.csv", "trade_log_ai_hardware___cloud.csv")] + [TestCase("trade log: AI & Cloud.csv", "trade log_ AI _ Cloud.csv")] + [TestCase("file.tar.gz", "file_tar.gz")] + [TestCase("résumé.pdf", "r_sum_.pdf")] + [TestCase("a..", "a__")] + [TestCase("100%.json", "100_.json")] + public void UnsupportedKeysAreSanitizedOnEveryReadAndSave(string key, string sanitizedKey) { - Assert.AreEqual(isSupported, ObjectStore.IsSupportedKey(key)); - if (isSupported) - { - Assert.DoesNotThrow(() => ObjectStore.ValidateKey(key)); - Assert.IsNull(ObjectStore.GetKeyIssues(key)); - } - else - { - var error = Assert.Throws(() => ObjectStore.ValidateKey(key)); - var issues = ObjectStore.GetKeyIssues(key); - Assert.IsNotNull(issues); - Assert.IsTrue(error.Message.EndsWith(issues), error.Message); - } + Assert.IsTrue(_store.Save(key, "abc")); + + Assert.IsTrue(_store.Keys.Contains(sanitizedKey), string.Join(", ", _store.Keys)); + Assert.AreEqual(key == sanitizedKey, _store.Keys.Contains(key)); + Assert.IsTrue(_store.ContainsKey(key)); + Assert.IsTrue(_store.ContainsKey(sanitizedKey)); + Assert.AreEqual("abc", _store.Read(key)); + Assert.AreEqual("abc", _store.Read(sanitizedKey)); + + Assert.IsTrue(_store.Delete(key)); + Assert.IsFalse(_store.ContainsKey(sanitizedKey)); } [Test] - public void UnsupportedKeyErrorStatesTheKeyRules() + public void UnsupportedKeysAreWarnedAboutOnceStatingTheIssueAndTheKeyUsed() { - var error = Assert.Throws( - () => _store.SaveBytes("trade_log_ai_hardware_&_cloud.csv", new byte[] { 1 })); + var warnings = new List(); + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + + Assert.IsTrue(store.Save("trade log: AI & Cloud.csv", "abc")); + Assert.IsTrue(store.ContainsKey("trade log: AI & Cloud.csv")); + Assert.AreEqual("abc", store.Read("trade log: AI & Cloud.csv")); + Assert.IsTrue(store.Save("trade log: AI & Cloud.csv", "abcd")); + Assert.IsTrue(store.Save("supported.csv", "abc")); + + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); + Assert.AreEqual("ObjectStore: key 'trade log: AI & Cloud.csv' is not supported: unsupported characters ':', '&'. " + + "Keys may only contain english letters, numbers, spaces and the characters '/', '\\', '_', '#', '-', '$' and '=', " + + "plus at most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv'. " + + "Using 'trade log_ AI _ Cloud.csv' instead, now and on every other read or save of this key. " + + "Any other unsupported key is sanitized the same way, without further warnings", warnings[0]); + + // the next unsupported key is sanitized silently + Assert.IsFalse(store.ContainsKey("trade log: AI.v2.csv")); + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); - Assert.IsTrue(error.Message.Contains("path is not supported: 'trade_log_ai_hardware_&_cloud.csv': unsupported character '&'."), error.Message); - Assert.IsTrue(error.Message.Contains("keys may only contain english letters, numbers, spaces and the characters"), error.Message); - Assert.IsTrue(error.Message.Contains("SanitizeKey"), error.Message); + store.Delete("trade log: AI & Cloud.csv"); + store.Delete("supported.csv"); } [TestCase("trade log: AI & Cloud.csv", "unsupported characters ':', '&'")] @@ -624,56 +644,90 @@ public void UnsupportedKeyErrorStatesTheKeyRules() [TestCase("folder.v1/file", "the extension 'v1/file' may only contain letters and numbers")] [TestCase("trade log: AI.v2.csv", "unsupported character ':'; it has 2 '.' but at most one '.extension' is allowed")] [TestCase(".", "it has no name before the '.'")] - [TestCase("", "the key is empty")] - [TestCase(null, "the key is empty")] - public void UnsupportedKeyErrorStatesTheIssue(string key, string expectedIssue) + public void UnsupportedKeyWarningStatesTheIssue(string key, string expectedIssue) { - var error = Assert.Throws(() => ObjectStore.ValidateKey(key)); + var warnings = new List(); + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + + Assert.IsFalse(store.ContainsKey(key)); - Assert.IsTrue(error.Message.Contains($"'{key}': {expectedIssue}."), error.Message); + Assert.AreEqual(1, warnings.Count); + Assert.That(warnings[0], Does.StartWith($"ObjectStore: key '{key}' is not supported: {expectedIssue}.")); } - [TestCase("my_key/file.csv", "my_key/file.csv")] - [TestCase("trade_log_ai_hardware_&_cloud.csv", "trade_log_ai_hardware___cloud.csv")] - [TestCase("trade log: AI & Cloud.csv", "trade log_ AI _ Cloud.csv")] - [TestCase("file.tar.gz", "file_tar.gz")] - [TestCase("résumé.pdf", "r_sum_.pdf")] - [TestCase("a..", "a__")] - [TestCase("100%.json", "100_.json")] - public void SanitizeKeyReturnsAStorableKey(string key, string expectedSanitizedKey) + [TestCase("")] + [TestCase(null)] + public void EmptyAndNullKeysAreRejected(string key) { - var sanitized = ObjectStore.SanitizeKey(key); + var warnings = new List(); + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); - Assert.AreEqual(expectedSanitizedKey, sanitized); - Assert.IsTrue(ObjectStore.IsSupportedKey(sanitized)); - Assert.IsTrue(_store.SaveBytes(sanitized, new byte[] { 1 })); + var error = Assert.Catch(() => store.Save(key, "abc")); - _store.Delete(sanitized); + Assert.IsEmpty(warnings); + if (key != null) + { + Assert.That(error.Message, Does.Contain("path is not supported: '': the key is empty.")); + } } - [TestCase(null)] - [TestCase("")] - public void SanitizeKeyThrowsOnNullOrEmptyKey(string key) + [Test] + public void SanitizationIsLossyAndDistinctKeysCanCollide() { - Assert.Throws(() => ObjectStore.SanitizeKey(key)); + // both sanitize to 'trade_log_ai_cloud.csv': the second save silently overwrites the first + Assert.IsTrue(_store.Save("trade_log_ai&cloud.csv", "first")); + Assert.IsTrue(_store.Save("trade_log_ai:cloud.csv", "second")); + + Assert.AreEqual("second", _store.Read("trade_log_ai&cloud.csv")); + Assert.AreEqual("second", _store.Read("trade_log_ai_cloud.csv")); + Assert.AreEqual(1, _store.Keys.Count(key => key.StartsWith("trade_log_ai"))); + + _store.Delete("trade_log_ai_cloud.csv"); + } + + [Test] + public void LocalObjectStoreRejectsUnsupportedKeysStatingTheIssue() + { + // the implementation is the last line of defense, only the wrapper sanitizes + using var store = new TestLocalObjectStore(); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }); + + var error = Assert.Throws(() => store.SaveBytes("trade_log_ai_hardware_&_cloud.csv", new byte[] { 1 })); + + Assert.AreEqual("ObjectStore: path is not supported: 'trade_log_ai_hardware_&_cloud.csv': unsupported character '&'. " + + "Keys may only contain english letters, numbers, spaces and the characters '/', '\\', '_', '#', '-', '$' and '=', " + + "plus at most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv'", error.Message); } [Test] - public void SanitizeKeyIsLossyAndDistinctNamesCanCollide() + public void AlgorithmWrapsTheStoreAndWarnsThroughDebug() { - var first = ObjectStore.SanitizeKey("trade_log_ai&cloud.csv"); - var second = ObjectStore.SanitizeKey("trade_log_ai:cloud.csv"); + using var localStore = new TestLocalObjectStore(); + localStore.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }); + var algorithm = new QCAlgorithm(); + + algorithm.SetObjectStore(localStore); + + Assert.AreEqual(typeof(ObjectStore), algorithm.ObjectStore.GetType()); + Assert.IsTrue(algorithm.ObjectStore.Save("trade_log_ai_hardware_&_cloud.csv", "abc")); + Assert.IsTrue(algorithm.DebugMessages.TryDequeue(out var warning)); + Assert.That(warning, Does.Contain("ObjectStore: key 'trade_log_ai_hardware_&_cloud.csv' is not supported: unsupported character '&'.")); + Assert.That(warning, Does.Contain("Using 'trade_log_ai_hardware___cloud.csv' instead")); + + algorithm.ObjectStore.Delete("trade_log_ai_hardware_&_cloud.csv"); + } - Assert.AreEqual("trade_log_ai_cloud.csv", first); - Assert.AreEqual(first, second); - Assert.AreEqual(first, ObjectStore.SanitizeKey("trade_log_ai_cloud.csv")); + [Test] + public void AlgorithmKeepsAnAlreadyWrappedStore() + { + using var pythonStore = new PythonObjectStore(new TestLocalObjectStore()); + var algorithm = new QCAlgorithm(); - // the second save silently overwrites the first: collision checks are up to the caller - Assert.IsTrue(_store.Save(first, "first")); - Assert.IsTrue(_store.Save(second, "second")); - Assert.AreEqual("second", _store.Read(first)); + algorithm.SetObjectStore(pythonStore); - _store.Delete(first); + Assert.AreSame(pythonStore, algorithm.ObjectStore); } [Test] @@ -688,20 +742,13 @@ public void SaveTextAndReadTextAreAliasesOfSaveAndRead() [TestCase("my_key", "./LocalObjectStoreTests/my_key")] [TestCase("test/123", "./LocalObjectStoreTests/test/123")] - [TestCase("**abc**", null)] - [TestCase("", null)] - [TestCase("|", null)] + [TestCase("**abc**", "./LocalObjectStoreTests/__abc__")] + [TestCase("", "./LocalObjectStoreTests/_random_")] + [TestCase("|", "./LocalObjectStoreTests/_")] public void GetFilePathReturnsFileName(string key, string expectedRelativePath) { - if (expectedRelativePath == null) - { - Assert.Throws(() => _store.GetFilePath(key)); - } - else - { - var expectedPath = Path.GetFullPath(expectedRelativePath).Replace("\\", "/"); - Assert.AreEqual(expectedPath, _store.GetFilePath(key).Replace("\\", "/")); - } + var expectedPath = Path.GetFullPath(expectedRelativePath).Replace("\\", "/"); + Assert.AreEqual(expectedPath, _store.GetFilePath(key).Replace("\\", "/")); } [Test] @@ -1478,11 +1525,15 @@ def test(object_store): } [Test] - public void PythonSanitizeKeyIsReachableFromTheInstance() + public void PythonHelpersSanitizeUnsupportedKeys() { RunPythonTest(@" def test(object_store): - assert object_store.sanitize_key('trade_log_ai_hardware_&_cloud.csv') == 'trade_log_ai_hardware___cloud.csv' + object_store.save_json('ergonomics/trade log: AI & Cloud.json', {'a': 1}) + assert 'ergonomics/trade log_ AI _ Cloud.json' in list(object_store.keys), list(object_store.keys) + assert object_store.read_json('ergonomics/trade log: AI & Cloud.json') == {'a': 1} + object_store.delete('ergonomics/trade log: AI & Cloud.json') + assert not object_store.contains_key('ergonomics/trade log_ AI _ Cloud.json') "); } diff --git a/Tests/Python/AlgorithmPythonWrapperTests.cs b/Tests/Python/AlgorithmPythonWrapperTests.cs index f2b406c9482c..902fcb37883c 100644 --- a/Tests/Python/AlgorithmPythonWrapperTests.cs +++ b/Tests/Python/AlgorithmPythonWrapperTests.cs @@ -21,6 +21,9 @@ using System.Collections.Generic; using System.IO; using QuantConnect.Orders; +using QuantConnect.Lean.Engine.Storage; +using QuantConnect.Packets; +using QuantConnect.Storage; namespace QuantConnect.Tests.Python { @@ -149,6 +152,30 @@ public void CallOnEndOfDayExceptionWithParameter() } } + [Test] + public void PythonAlgorithmGetsAPythonObjectStoreWarningThroughDebug() + { + using (Py.GIL()) + { + var algorithm = GetAlgorithm("def on_end_of_algorithm(self): self.object_store.save_json('trade log: AI & Cloud.json', {'a': 1})"); + using var localStore = new LocalObjectStore(); + localStore.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + + algorithm.SetObjectStore(localStore); + + Assert.IsInstanceOf(algorithm.ObjectStore); + // the Python helpers are reachable through self.object_store and the key warning lands in the algorithm's log + algorithm.OnEndOfAlgorithm(); + Assert.Null(algorithm.RunTimeError); + Assert.IsTrue(localStore.Keys.Contains("trade log_ AI _ Cloud.json"), string.Join(", ", localStore.Keys)); + Assert.IsTrue(algorithm.BaseAlgorithm.DebugMessages.TryDequeue(out var warning)); + Assert.That(warning, Does.Contain("ObjectStore: key 'trade log: AI & Cloud.json' is not supported: unsupported characters ':', '&'.")); + Assert.That(warning, Does.Contain("Using 'trade log_ AI _ Cloud.json' instead")); + + localStore.Delete("trade log_ AI _ Cloud.json"); + } + } + private AlgorithmPythonWrapper GetAlgorithm(string code) { code = $"{_baseCode}{Environment.NewLine} {code}"; From 9745927b422ae349f7caa566817db0b3f08c96a6 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 3 Nov 2026 14:43:10 -0400 Subject: [PATCH 08/14] Use existing keys as given and warn on every overwrite through a sanitized key --- Common/Storage/ObjectStore.cs | 72 ++++++++--- Research/QuantBook.cs | 2 +- Tests/Common/Storage/LocalObjectStoreTests.cs | 116 ++++++++++++++++-- 3 files changed, 158 insertions(+), 32 deletions(-) diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index da90b5fd529b..01a4c38eb0cd 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -33,8 +33,9 @@ namespace QuantConnect.Storage /// /// Helper class for easier access to methods. A key that does not follow the key /// format (english letters, numbers, spaces and the characters '/', '\', '_', '#', '-', '$' and '=', plus at - /// most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv') is - /// replaced by its sanitized version on every read and save, with a single warning the first time it happens + /// most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv') is saved + /// under its sanitized version and found by either name, with a warning the first time it happens and whenever + /// the sanitized name overwrites an existing object. An object already stored under an unsupported name is used as is /// public class ObjectStore : IObjectStore { @@ -46,8 +47,8 @@ public class ObjectStore : IObjectStore private readonly IObjectStore _store; private readonly Action _warn; private int _warned; - // every key used so far, mapped to the key the store is asked for, so each one is checked once - private readonly ConcurrentDictionary _keys = new(); + // the unsupported keys this store saved: overwriting what they produced is an update, not a collision + private readonly ConcurrentDictionary _savedKeys = new(); /// /// Gets the maximum storage limit in bytes @@ -72,8 +73,7 @@ public event EventHandler ErrorRaised /// Initializes a new instance of the class /// /// The instance to wrap - /// Receives the single warning logged the first time an unsupported key is sanitized, - /// the engine log by default + /// Receives the warnings about sanitized keys, the engine log by default public ObjectStore(IObjectStore store, Action warn = null) { _store = store; @@ -110,7 +110,7 @@ public void Initialize(int userId, int projectId, string userToken, Controls con /// True if the key was found public bool ContainsKey(string path) { - return _store.ContainsKey(GetKey(path)); + return _store.ContainsKey(KeyToRead(path)); } /// @@ -120,7 +120,7 @@ public bool ContainsKey(string path) /// A byte array containing the data public byte[] ReadBytes(string path) { - return _store.ReadBytes(GetKey(path)); + return _store.ReadBytes(KeyToRead(path)); } /// @@ -131,7 +131,7 @@ public byte[] ReadBytes(string path) /// True if the save operation was successful public bool SaveBytes(string path, byte[] contents) { - return _store.SaveBytes(GetKey(path), contents); + return _store.SaveBytes(KeyToSave(path), contents); } /// @@ -141,7 +141,7 @@ public bool SaveBytes(string path, byte[] contents) /// True if the delete operation was successful public bool Delete(string path) { - return _store.Delete(GetKey(path)); + return _store.Delete(KeyToRead(path)); } /// @@ -151,7 +151,7 @@ public bool Delete(string path) /// The path for the file public string GetFilePath(string path) { - return _store.GetFilePath(GetKey(path)); + return _store.GetFilePath(KeyToRead(path)); } /// @@ -357,30 +357,62 @@ internal static void ValidateKey(string key) } /// - /// The key the store is asked for: the given one when it follows the key format, else its sanitized - /// version, warning only the first time any key is sanitized. Null and empty keys are left for the store to reject + /// The key a save goes to: the given one when it follows the key format, else its sanitized version. + /// Null and empty keys are left for the store to reject /// - private string GetKey(string path) + private string KeyToSave(string path) { if (string.IsNullOrEmpty(path)) { return path; } - if (_keys.TryGetValue(path, out var key)) + var issues = GetKeyIssues(path); + if (issues == null) { - return key; + return path; } + var key = SanitizeKey(path); + WarnOnce(path, issues, key); + // the rejection used to prevent this: silently replacing an object saved under another key + if (!_savedKeys.ContainsKey(path) && _store.ContainsKey(key)) + { + _warn($"ObjectStore: key '{path}' is not supported and is saved as '{key}', overwriting the existing object '{key}'"); + } + _savedKeys[path] = key; + return key; + } + + /// + /// The key the other operations use: the given one when it follows the key format or the store already has + /// it, else its sanitized version, so an object is found by the key it was saved with + /// + private string KeyToRead(string path) + { + if (string.IsNullOrEmpty(path)) + { + return path; + } var issues = GetKeyIssues(path); - key = issues == null ? path : SanitizeKey(path); - if (_keys.TryAdd(path, key) && issues != null && Interlocked.Exchange(ref _warned, 1) == 0) + if (issues == null || _store.ContainsKey(path)) { - _warn($"ObjectStore: key '{path}' is not supported: {issues}. Using '{key}' instead, now and on every" + - " other read or save of this key. Any other unsupported key is sanitized the same way, without further warnings"); + return path; } + + var key = SanitizeKey(path); + WarnOnce(path, issues, key); return key; } + private void WarnOnce(string path, string issues, string key) + { + if (Interlocked.Exchange(ref _warned, 1) == 0) + { + _warn($"ObjectStore: key '{path}' is not supported: {issues}. Using '{key}' instead. Any other unsupported key" + + " is sanitized the same way, with a warning only when it overwrites an existing object"); + } + } + private static bool IsSupportedKey(string key) { return !string.IsNullOrEmpty(key) && SupportedKeyRegex.IsMatch(key) diff --git a/Research/QuantBook.cs b/Research/QuantBook.cs index 37b5a132d5e3..ad9678d43726 100644 --- a/Research/QuantBook.cs +++ b/Research/QuantBook.cs @@ -166,7 +166,7 @@ public QuantBook() : base() StorageAccess = Config.GetValue("storage-permissions", new Packets.StoragePermissions()) }, AlgorithmMode.Research); - // wrapped here so key warnings reach the console: a notebook has no debug message consumer + // key warnings go to the engine log, like every other QuantBook message: nothing drains the debug messages here SetObjectStore(_isPythonNotebook ? new PythonObjectStore(algorithmHandlers.ObjectStore) : new ObjectStore(algorithmHandlers.ObjectStore)); diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index d5baca3df3bf..7276b9234d90 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -27,6 +27,8 @@ using System.Threading.Tasks; using System.Collections.Generic; using QuantConnect.Lean.Engine.Storage; +using QuantConnect.Lean.Engine.DataFeeds.Transport; +using System.Reflection; using Python.Runtime; namespace QuantConnect.Tests.Common.Storage @@ -594,7 +596,7 @@ public void KeyNotFoundErrorTruncatesTheAvailableKeysListAtTen() [TestCase("résumé.pdf", "r_sum_.pdf")] [TestCase("a..", "a__")] [TestCase("100%.json", "100_.json")] - public void UnsupportedKeysAreSanitizedOnEveryReadAndSave(string key, string sanitizedKey) + public void UnsupportedKeysAreSanitizedOnSaveAndFoundByTheKeyTheyWereSavedWith(string key, string sanitizedKey) { Assert.IsTrue(_store.Save(key, "abc")); @@ -626,8 +628,8 @@ public void UnsupportedKeysAreWarnedAboutOnceStatingTheIssueAndTheKeyUsed() Assert.AreEqual("ObjectStore: key 'trade log: AI & Cloud.csv' is not supported: unsupported characters ':', '&'. " + "Keys may only contain english letters, numbers, spaces and the characters '/', '\\', '_', '#', '-', '$' and '=', " + "plus at most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv'. " + - "Using 'trade log_ AI _ Cloud.csv' instead, now and on every other read or save of this key. " + - "Any other unsupported key is sanitized the same way, without further warnings", warnings[0]); + "Using 'trade log_ AI _ Cloud.csv' instead. Any other unsupported key is sanitized the same way, " + + "with a warning only when it overwrites an existing object", warnings[0]); // the next unsupported key is sanitized silently Assert.IsFalse(store.ContainsKey("trade log: AI.v2.csv")); @@ -673,18 +675,110 @@ public void EmptyAndNullKeysAreRejected(string key) } } + [TestCase("model.tar.gz")] + [TestCase("SPY (1).csv")] + [TestCase("BRK.B.csv")] + public void ExistingUnsupportedKeysAreUsedAsGiven(string key) + { + // an object put in the store by other means (uploaded, copied into the folder) under an unsupported name + var warnings = new List(); + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + var filePath = Path.Combine(TestStorageRoot, key); + File.WriteAllText(filePath, "abc"); + + Assert.IsTrue(store.ContainsKey(key)); + Assert.AreEqual("abc", store.Read(key)); + Assert.IsTrue(store.Keys.Contains(key), string.Join(", ", store.Keys)); + Assert.AreEqual(Path.GetFullPath(filePath), Path.GetFullPath(store.GetFilePath(key))); + Assert.IsEmpty(warnings, string.Join("\n", warnings)); + Assert.IsTrue(store.Delete(key)); + Assert.IsFalse(File.Exists(filePath)); + + // gone from the store, the key falls back to its sanitized name like any other unsupported key + Assert.IsFalse(store.ContainsKey(key)); + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); + Assert.That(warnings[0], Does.StartWith($"ObjectStore: key '{key}' is not supported:")); + } + + [Test] + public void CustomDataIsReadFromAnExistingUnsupportedKey() + { + using var store = new ObjectStore(new TestLocalObjectStore()); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + File.WriteAllText(Path.Combine(TestStorageRoot, "BRK.B.csv"), "2013-10-07,167.42\n2013-10-08,165.48\n"); + + // the data feed reads custom data through the algorithm's store + using var reader = new ObjectStoreSubscriptionStreamReader(store, "BRK.B.csv"); + + Assert.IsFalse(reader.EndOfStream); + Assert.AreEqual("2013-10-07,167.42", reader.ReadLine()); + Assert.AreEqual("2013-10-08,165.48", reader.ReadLine()); + store.Delete("BRK.B.csv"); + } + + [Test] + public void SavingAnUnsupportedKeyOverAnotherObjectWarnsEveryTime() + { + var warnings = new List(); + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.IsTrue(store.Save("trade_log_ai_cloud.csv", "existing")); + Assert.IsEmpty(warnings); + + // sanitizes to the existing object's key: the general warning, then the overwrite one + Assert.IsTrue(store.Save("trade_log_ai&cloud.csv", "first")); + Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); + Assert.That(warnings[0], Does.StartWith("ObjectStore: key 'trade_log_ai&cloud.csv' is not supported: unsupported character '&'.")); + Assert.AreEqual("ObjectStore: key 'trade_log_ai&cloud.csv' is not supported and is saved as 'trade_log_ai_cloud.csv', " + + "overwriting the existing object 'trade_log_ai_cloud.csv'", warnings[1]); + + // updating its own object is silent + Assert.IsTrue(store.Save("trade_log_ai&cloud.csv", "second")); + Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); + + // a different unsupported key sanitizing to the same object warns again + Assert.IsTrue(store.Save("trade_log_ai:cloud.csv", "third")); + Assert.AreEqual(3, warnings.Count, string.Join("\n", warnings)); + Assert.AreEqual("ObjectStore: key 'trade_log_ai:cloud.csv' is not supported and is saved as 'trade_log_ai_cloud.csv', " + + "overwriting the existing object 'trade_log_ai_cloud.csv'", warnings[2]); + Assert.AreEqual("third", store.Read("trade_log_ai_cloud.csv")); + Assert.AreEqual(1, store.Keys.Count(key => key.StartsWith("trade_log_ai"))); + + store.Delete("trade_log_ai_cloud.csv"); + } + [Test] - public void SanitizationIsLossyAndDistinctKeysCanCollide() + public void QuantBookWarnsThroughTheEngineLog() { - // both sanitize to 'trade_log_ai_cloud.csv': the second save silently overwrites the first - Assert.IsTrue(_store.Save("trade_log_ai&cloud.csv", "first")); - Assert.IsTrue(_store.Save("trade_log_ai:cloud.csv", "second")); + var traces = new List(); + Log.LogHandler = new FunctionalLogHandler(null, traces.Add, null); + var quantBook = new QuantBook(); + Assert.AreEqual(typeof(ObjectStore), quantBook.ObjectStore.GetType()); - Assert.AreEqual("second", _store.Read("trade_log_ai&cloud.csv")); - Assert.AreEqual("second", _store.Read("trade_log_ai_cloud.csv")); - Assert.AreEqual(1, _store.Keys.Count(key => key.StartsWith("trade_log_ai"))); + Assert.IsTrue(quantBook.ObjectStore.Save("trade log: AI & Cloud.csv", "abc")); - _store.Delete("trade_log_ai_cloud.csv"); + Assert.AreEqual(1, traces.Count(trace => trace.Contains("ObjectStore: key 'trade log: AI & Cloud.csv' is not supported: unsupported characters ':', '&'.")), + string.Join("\n", traces)); + quantBook.ObjectStore.Delete("trade log: AI & Cloud.csv"); + } + + [Test] + public void QuantBookUsesThePythonObjectStoreInPythonNotebooks() + { + // the notebook kernel is detected once, in the static constructor + var isPythonNotebook = typeof(QuantBook).GetField("_isPythonNotebook", BindingFlags.NonPublic | BindingFlags.Static); + isPythonNotebook.SetValue(null, true); + try + { + var quantBook = new QuantBook(); + + Assert.IsInstanceOf(quantBook.ObjectStore); + } + finally + { + isPythonNotebook.SetValue(null, false); + } } [Test] From cc1b68e5be9e3e13d59c6ff828138208a99b40e9 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 3 Nov 2026 15:04:03 -0400 Subject: [PATCH 09/14] Name a key starting with more than one dot as its own issue --- Common/Storage/ObjectStore.cs | 6 +++++- Tests/Common/Storage/LocalObjectStoreTests.cs | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index 01a4c38eb0cd..1895e4959858 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -468,10 +468,14 @@ private static List FindKeyIssues(string key) issues.Add($"the extension '{extension}' may only contain letters and numbers"); } } - if (name.Length == 0 || name[0] == '.') + if (name.Length == 0) { issues.Add("it has no name before the '.'"); } + else if (key.StartsWith("..", StringComparison.Ordinal)) + { + issues.Add("it starts with more than one '.'"); + } if (key.Count(c => c == '/') > 100 || key.Count(c => c == '\\') > 100) { diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index 7276b9234d90..ba1be4290287 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -596,6 +596,7 @@ public void KeyNotFoundErrorTruncatesTheAvailableKeysListAtTen() [TestCase("résumé.pdf", "r_sum_.pdf")] [TestCase("a..", "a__")] [TestCase("100%.json", "100_.json")] + [TestCase("..model.pkl", "__model.pkl")] public void UnsupportedKeysAreSanitizedOnSaveAndFoundByTheKeyTheyWereSavedWith(string key, string sanitizedKey) { Assert.IsTrue(_store.Save(key, "abc")); @@ -646,6 +647,7 @@ public void UnsupportedKeysAreWarnedAboutOnceStatingTheIssueAndTheKeyUsed() [TestCase("folder.v1/file", "the extension 'v1/file' may only contain letters and numbers")] [TestCase("trade log: AI.v2.csv", "unsupported character ':'; it has 2 '.' but at most one '.extension' is allowed")] [TestCase(".", "it has no name before the '.'")] + [TestCase("..model.pkl", "it starts with more than one '.'")] public void UnsupportedKeyWarningStatesTheIssue(string key, string expectedIssue) { var warnings = new List(); From 07bb4f5ce9967beb8a9276d7b80b15464ba5408b Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 3 Nov 2026 16:50:46 -0400 Subject: [PATCH 10/14] Warn on every save to an existing unsupported key and follow the saved key on reads Saving to an unsupported key the store already holds as given goes to the sanitized name with a warning every time. Reads of a key this store saved follow the save while the saved object exists, and the mapping is recorded only once the store accepts the save. --- Common/Storage/ObjectStore.cs | 61 +++++++----- Tests/Common/Storage/LocalObjectStoreTests.cs | 99 ++++++++++++++++++- 2 files changed, 137 insertions(+), 23 deletions(-) diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index 1895e4959858..a36eea373666 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -35,7 +35,8 @@ namespace QuantConnect.Storage /// format (english letters, numbers, spaces and the characters '/', '\', '_', '#', '-', '$' and '=', plus at /// most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv') is saved /// under its sanitized version and found by either name, with a warning the first time it happens and whenever - /// the sanitized name overwrites an existing object. An object already stored under an unsupported name is used as is + /// an object already exists under the key or its sanitized name. An object already stored under an unsupported + /// name is read as is; a save to that name goes to the sanitized name, warning every time /// public class ObjectStore : IObjectStore { @@ -47,7 +48,7 @@ public class ObjectStore : IObjectStore private readonly IObjectStore _store; private readonly Action _warn; private int _warned; - // the unsupported keys this store saved: overwriting what they produced is an update, not a collision + // the unsupported keys this store saved and where they went: reads follow them, and overwriting them is an update private readonly ConcurrentDictionary _savedKeys = new(); /// @@ -131,7 +132,16 @@ public byte[] ReadBytes(string path) /// True if the save operation was successful public bool SaveBytes(string path, byte[] contents) { - return _store.SaveBytes(KeyToSave(path), contents); + var key = KeyToSave(path); + if (!_store.SaveBytes(key, contents)) + { + return false; + } + if (key != path) + { + _savedKeys[path] = key; + } + return true; } /// @@ -362,30 +372,29 @@ internal static void ValidateKey(string key) /// private string KeyToSave(string path) { - if (string.IsNullOrEmpty(path)) - { - return path; - } - var issues = GetKeyIssues(path); - if (issues == null) + if (string.IsNullOrEmpty(path) || IsSupportedKey(path)) { return path; } var key = SanitizeKey(path); - WarnOnce(path, issues, key); - // the rejection used to prevent this: silently replacing an object saved under another key - if (!_savedKeys.ContainsKey(path) && _store.ContainsKey(key)) + WarnOnce(path, key); + if (_store.ContainsKey(path)) { + // the object under the name as given stays: reads of it from another store still return it + _warn($"ObjectStore: key '{path}' is not supported: the object stored under that name is left as is, this save goes to '{key}'"); + } + else if (!_savedKeys.ContainsKey(path) && _store.ContainsKey(key)) + { + // the rejection used to prevent this: silently replacing an object saved under another key _warn($"ObjectStore: key '{path}' is not supported and is saved as '{key}', overwriting the existing object '{key}'"); } - _savedKeys[path] = key; return key; } /// - /// The key the other operations use: the given one when it follows the key format or the store already has - /// it, else its sanitized version, so an object is found by the key it was saved with + /// The key the other operations use: where this store saved it while that object exists, else the given one when + /// it follows the key format or the store already has it, else its sanitized version, so an object is found by the key it was saved with /// private string KeyToRead(string path) { @@ -393,23 +402,31 @@ private string KeyToRead(string path) { return path; } - var issues = GetKeyIssues(path); - if (issues == null || _store.ContainsKey(path)) + if (_savedKeys.TryGetValue(path, out var key)) + { + if (_store.ContainsKey(key)) + { + return key; + } + // what this store saved under the key is gone: the key resolves like a fresh store's again + _savedKeys.TryRemove(path, out _); + } + if (IsSupportedKey(path) || _store.ContainsKey(path)) { return path; } - var key = SanitizeKey(path); - WarnOnce(path, issues, key); + key = SanitizeKey(path); + WarnOnce(path, key); return key; } - private void WarnOnce(string path, string issues, string key) + private void WarnOnce(string path, string key) { if (Interlocked.Exchange(ref _warned, 1) == 0) { - _warn($"ObjectStore: key '{path}' is not supported: {issues}. Using '{key}' instead. Any other unsupported key" + - " is sanitized the same way, with a warning only when it overwrites an existing object"); + _warn($"ObjectStore: key '{path}' is not supported: {GetKeyIssues(path)}. Using '{key}' instead. Any other unsupported" + + " key is sanitized the same way, with a warning only when an object already exists under it or under its sanitized name"); } } diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index ba1be4290287..af13420c516e 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -630,7 +630,7 @@ public void UnsupportedKeysAreWarnedAboutOnceStatingTheIssueAndTheKeyUsed() "Keys may only contain english letters, numbers, spaces and the characters '/', '\\', '_', '#', '-', '$' and '=', " + "plus at most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv'. " + "Using 'trade log_ AI _ Cloud.csv' instead. Any other unsupported key is sanitized the same way, " + - "with a warning only when it overwrites an existing object", warnings[0]); + "with a warning only when an object already exists under it or under its sanitized name", warnings[0]); // the next unsupported key is sanitized silently Assert.IsFalse(store.ContainsKey("trade log: AI.v2.csv")); @@ -703,6 +703,103 @@ public void ExistingUnsupportedKeysAreUsedAsGiven(string key) Assert.That(warnings[0], Does.StartWith($"ObjectStore: key '{key}' is not supported:")); } + [Test] + public void SavingToAnExistingUnsupportedKeyGoesToTheSanitizedNameWithAWarningEveryTime() + { + // an uploaded object under an unsupported name, which the algorithm loads, retrains and saves back + var filePath = Path.Combine(TestStorageRoot, "model.tar.gz"); + File.WriteAllText(filePath, "old"); + var warnings = new List(); + using (var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add)) + { + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.AreEqual("old", store.Read("model.tar.gz")); + + Assert.IsTrue(store.Save("model.tar.gz", "new")); + + Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); + Assert.That(warnings[0], Does.StartWith("ObjectStore: key 'model.tar.gz' is not supported: it has 2 '.' but at most one '.extension' is allowed.")); + Assert.AreEqual("ObjectStore: key 'model.tar.gz' is not supported: the object stored under that name is left as is," + + " this save goes to 'model_tar.gz'", warnings[1]); + // read after write within the run follows the save + Assert.AreEqual("new", store.Read("model.tar.gz")); + Assert.AreEqual("new", store.Read("model_tar.gz")); + Assert.AreEqual("old", File.ReadAllText(filePath)); + Assert.IsTrue(store.Keys.Contains("model.tar.gz") && store.Keys.Contains("model_tar.gz"), string.Join(", ", store.Keys)); + + // every such save warns, not only the first + Assert.IsTrue(store.Save("model.tar.gz", "newer")); + Assert.AreEqual(3, warnings.Count, string.Join("\n", warnings)); + Assert.AreEqual(warnings[1], warnings[2]); + Assert.AreEqual("newer", store.Read("model.tar.gz")); + } + + // a fresh store, the next run: the object under the name as given comes first again, and the save warns again + warnings.Clear(); + using (var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add)) + { + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.AreEqual("old", store.Read("model.tar.gz")); + Assert.IsEmpty(warnings); + + Assert.IsTrue(store.Save("model.tar.gz", "third")); + + Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); + Assert.That(warnings[1], Does.Contain("the object stored under that name is left as is, this save goes to 'model_tar.gz'")); + Assert.AreEqual("third", store.Read("model.tar.gz")); + Assert.AreEqual("third", store.Read("model_tar.gz")); + + // within the run the key follows the save: the first delete removes it and the original is back + Assert.IsTrue(store.Delete("model.tar.gz")); + Assert.IsFalse(store.ContainsKey("model_tar.gz")); + Assert.AreEqual("old", File.ReadAllText(filePath)); + Assert.AreEqual("old", store.Read("model.tar.gz")); + Assert.IsTrue(store.Delete("model.tar.gz")); + Assert.IsFalse(store.ContainsKey("model.tar.gz")); + } + Assert.IsFalse(File.Exists(filePath)); + } + + [Test] + public void DeletingASavedUnsupportedKeyRemovesTheSavedObjectThenTheOriginal() + { + var filePath = Path.Combine(TestStorageRoot, "model.tar.gz"); + File.WriteAllText(filePath, "old"); + using var store = new ObjectStore(new TestLocalObjectStore(), _ => { }); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.IsTrue(store.Save("model.tar.gz", "new")); + Assert.AreEqual("new", store.Read("model.tar.gz")); + + // the first delete removes what this store saved, and the key resolves like a fresh store's from then on + Assert.IsTrue(store.Delete("model.tar.gz")); + Assert.IsFalse(File.Exists(Path.Combine(TestStorageRoot, "model_tar.gz"))); + Assert.IsTrue(store.ContainsKey("model.tar.gz")); + Assert.AreEqual("old", store.Read("model.tar.gz")); + + Assert.IsTrue(store.Delete("model.tar.gz")); + Assert.IsFalse(store.ContainsKey("model.tar.gz")); + Assert.IsFalse(File.Exists(filePath)); + } + + [Test] + public void AFailedSaveToAnUnsupportedKeyDoesNotRedirectItsReads() + { + var filePath = Path.Combine(TestStorageRoot, "model.tar.gz"); + File.WriteAllText(filePath, "old"); + using var store = new ObjectStore(new TestLocalObjectStore(), _ => { }); + // the 3 bytes stored fill the storage: the save is rejected, not thrown, in backtesting + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1, StorageLimit = 3 }, AlgorithmMode.Backtesting); + + Assert.IsFalse(store.Save("model.tar.gz", "new")); + + Assert.IsTrue(store.ContainsKey("model.tar.gz")); + Assert.AreEqual("old", store.Read("model.tar.gz")); + Assert.IsFalse(store.ContainsKey("model_tar.gz")); + + Assert.IsTrue(store.Delete("model.tar.gz")); + Assert.IsFalse(File.Exists(filePath)); + } + [Test] public void CustomDataIsReadFromAnExistingUnsupportedKey() { From 003a855968a44f691d263272643af0e74e11c201 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 3 Nov 2026 17:34:39 -0400 Subject: [PATCH 11/14] Reject a save whose sanitized key already holds an object written from another key A save of an unsupported key goes to the sanitized key only when nothing is stored under it, or when this store wrote it from the same key. Saves through a different key and deletes forget that pairing, so later saves through the unsupported key are rejected with a warning until the sanitized key is free or written directly. --- Common/Storage/ObjectStore.cs | 66 +++++++++----- Tests/Common/Storage/LocalObjectStoreTests.cs | 89 +++++++++++++++---- 2 files changed, 112 insertions(+), 43 deletions(-) diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index a36eea373666..40fc01ae379c 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -34,9 +34,9 @@ namespace QuantConnect.Storage /// Helper class for easier access to methods. A key that does not follow the key /// format (english letters, numbers, spaces and the characters '/', '\', '_', '#', '-', '$' and '=', plus at /// most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv') is saved - /// under its sanitized version and found by either name, with a warning the first time it happens and whenever - /// an object already exists under the key or its sanitized name. An object already stored under an unsupported - /// name is read as is; a save to that name goes to the sanitized name, warning every time + /// under its sanitized version and found by either name, with a warning the first time it happens. A save whose + /// sanitized name already holds an object this store did not write from that key is rejected with a warning. An object + /// already stored under an unsupported name is read as is; a save to that name goes to the sanitized name, warning every time /// public class ObjectStore : IObjectStore { @@ -48,7 +48,7 @@ public class ObjectStore : IObjectStore private readonly IObjectStore _store; private readonly Action _warn; private int _warned; - // the unsupported keys this store saved and where they went: reads follow them, and overwriting them is an update + // the unsupported keys this store last wrote each sanitized name from: their saves are updates, and reads follow them private readonly ConcurrentDictionary _savedKeys = new(); /// @@ -132,11 +132,11 @@ public byte[] ReadBytes(string path) /// True if the save operation was successful public bool SaveBytes(string path, byte[] contents) { - var key = KeyToSave(path); - if (!_store.SaveBytes(key, contents)) + if (!TryGetKeyToSave(path, out var key) || !_store.SaveBytes(key, contents)) { return false; } + ForgetSavesTo(key, exceptFrom: path); if (key != path) { _savedKeys[path] = key; @@ -151,7 +151,13 @@ public bool SaveBytes(string path, byte[] contents) /// True if the delete operation was successful public bool Delete(string path) { - return _store.Delete(KeyToRead(path)); + var key = KeyToRead(path); + if (!_store.Delete(key)) + { + return false; + } + ForgetSavesTo(key); + return true; } /// @@ -367,29 +373,46 @@ internal static void ValidateKey(string key) } /// - /// The key a save goes to: the given one when it follows the key format, else its sanitized version. - /// Null and empty keys are left for the store to reject + /// The key a save goes to: the given one when it follows the key format, else its sanitized version, unless that + /// already holds an object this store did not last write from the given key. Null and empty keys are left for the store to reject /// - private string KeyToSave(string path) + private bool TryGetKeyToSave(string path, out string key) { + key = path; if (string.IsNullOrEmpty(path) || IsSupportedKey(path)) { - return path; + return true; } - var key = SanitizeKey(path); + key = SanitizeKey(path); WarnOnce(path, key); + if (!_savedKeys.ContainsKey(path) && _store.ContainsKey(key)) + { + // the rejection used to prevent this: silently replacing an object saved under another key + _warn($"ObjectStore: key '{path}' is not supported and its sanitized name '{key}' already holds another object:" + + $" the save is rejected. Delete '{key}' or save to it directly to replace it"); + return false; + } if (_store.ContainsKey(path)) { // the object under the name as given stays: reads of it from another store still return it _warn($"ObjectStore: key '{path}' is not supported: the object stored under that name is left as is, this save goes to '{key}'"); } - else if (!_savedKeys.ContainsKey(path) && _store.ContainsKey(key)) + return true; + } + + /// + /// Forgets the unsupported keys whose object under the given key is gone or replaced from another key + /// + private void ForgetSavesTo(string key, string exceptFrom = null) + { + foreach (var entry in _savedKeys) { - // the rejection used to prevent this: silently replacing an object saved under another key - _warn($"ObjectStore: key '{path}' is not supported and is saved as '{key}', overwriting the existing object '{key}'"); + if (entry.Value == key && entry.Key != exceptFrom) + { + _savedKeys.TryRemove(entry.Key, out _); + } } - return key; } /// @@ -402,14 +425,9 @@ private string KeyToRead(string path) { return path; } - if (_savedKeys.TryGetValue(path, out var key)) + if (_savedKeys.TryGetValue(path, out var key) && _store.ContainsKey(key)) { - if (_store.ContainsKey(key)) - { - return key; - } - // what this store saved under the key is gone: the key resolves like a fresh store's again - _savedKeys.TryRemove(path, out _); + return key; } if (IsSupportedKey(path) || _store.ContainsKey(path)) { @@ -426,7 +444,7 @@ private void WarnOnce(string path, string key) if (Interlocked.Exchange(ref _warned, 1) == 0) { _warn($"ObjectStore: key '{path}' is not supported: {GetKeyIssues(path)}. Using '{key}' instead. Any other unsupported" + - " key is sanitized the same way, with a warning only when an object already exists under it or under its sanitized name"); + " key is sanitized the same way; a save is rejected, with a warning, when its sanitized name already holds another object"); } } diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index af13420c516e..6d78d0049a7d 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -629,8 +629,8 @@ public void UnsupportedKeysAreWarnedAboutOnceStatingTheIssueAndTheKeyUsed() Assert.AreEqual("ObjectStore: key 'trade log: AI & Cloud.csv' is not supported: unsupported characters ':', '&'. " + "Keys may only contain english letters, numbers, spaces and the characters '/', '\\', '_', '#', '-', '$' and '=', " + "plus at most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv'. " + - "Using 'trade log_ AI _ Cloud.csv' instead. Any other unsupported key is sanitized the same way, " + - "with a warning only when an object already exists under it or under its sanitized name", warnings[0]); + "Using 'trade log_ AI _ Cloud.csv' instead. Any other unsupported key is sanitized the same way; " + + "a save is rejected, with a warning, when its sanitized name already holds another object", warnings[0]); // the next unsupported key is sanitized silently Assert.IsFalse(store.ContainsKey("trade log: AI.v2.csv")); @@ -734,7 +734,8 @@ public void SavingToAnExistingUnsupportedKeyGoesToTheSanitizedNameWithAWarningEv Assert.AreEqual("newer", store.Read("model.tar.gz")); } - // a fresh store, the next run: the object under the name as given comes first again, and the save warns again + // a fresh store, the next run: the object under the name as given comes first again, and the save finds + // the previous run's object under the sanitized name, which nothing ties to this key any more warnings.Clear(); using (var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add)) { @@ -742,10 +743,18 @@ public void SavingToAnExistingUnsupportedKeyGoesToTheSanitizedNameWithAWarningEv Assert.AreEqual("old", store.Read("model.tar.gz")); Assert.IsEmpty(warnings); - Assert.IsTrue(store.Save("model.tar.gz", "third")); + Assert.IsFalse(store.Save("model.tar.gz", "third")); Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); - Assert.That(warnings[1], Does.Contain("the object stored under that name is left as is, this save goes to 'model_tar.gz'")); + Assert.AreEqual("ObjectStore: key 'model.tar.gz' is not supported and its sanitized name 'model_tar.gz' already holds" + + " another object: the save is rejected. Delete 'model_tar.gz' or save to it directly to replace it", warnings[1]); + Assert.AreEqual("newer", store.Read("model_tar.gz")); + + // once the sanitized name is free the save goes there, warning about the original every time + Assert.IsTrue(store.Delete("model_tar.gz")); + Assert.IsTrue(store.Save("model.tar.gz", "third")); + Assert.AreEqual(3, warnings.Count, string.Join("\n", warnings)); + Assert.That(warnings[2], Does.Contain("the object stored under that name is left as is, this save goes to 'model_tar.gz'")); Assert.AreEqual("third", store.Read("model.tar.gz")); Assert.AreEqual("third", store.Read("model_tar.gz")); @@ -817,7 +826,7 @@ public void CustomDataIsReadFromAnExistingUnsupportedKey() } [Test] - public void SavingAnUnsupportedKeyOverAnotherObjectWarnsEveryTime() + public void SavingAnUnsupportedKeyWhoseSanitizedNameHoldsAnotherObjectIsRejectedWithAWarningEveryTime() { var warnings = new List(); using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); @@ -825,26 +834,68 @@ public void SavingAnUnsupportedKeyOverAnotherObjectWarnsEveryTime() Assert.IsTrue(store.Save("trade_log_ai_cloud.csv", "existing")); Assert.IsEmpty(warnings); - // sanitizes to the existing object's key: the general warning, then the overwrite one - Assert.IsTrue(store.Save("trade_log_ai&cloud.csv", "first")); + // sanitizes to the existing object's key: the general warning, then the rejection + Assert.IsFalse(store.Save("trade_log_ai&cloud.csv", "first")); Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); Assert.That(warnings[0], Does.StartWith("ObjectStore: key 'trade_log_ai&cloud.csv' is not supported: unsupported character '&'.")); - Assert.AreEqual("ObjectStore: key 'trade_log_ai&cloud.csv' is not supported and is saved as 'trade_log_ai_cloud.csv', " + - "overwriting the existing object 'trade_log_ai_cloud.csv'", warnings[1]); - - // updating its own object is silent + Assert.AreEqual("ObjectStore: key 'trade_log_ai&cloud.csv' is not supported and its sanitized name 'trade_log_ai_cloud.csv' " + + "already holds another object: the save is rejected. Delete 'trade_log_ai_cloud.csv' or save to it directly to replace it", warnings[1]); + Assert.AreEqual("existing", store.Read("trade_log_ai_cloud.csv")); + + // every such save is rejected with the warning, from this key or another one sanitizing to the same name + Assert.IsFalse(store.Save("trade_log_ai&cloud.csv", "first")); + Assert.IsFalse(store.Save("trade_log_ai:cloud.csv", "second")); + Assert.AreEqual(4, warnings.Count, string.Join("\n", warnings)); + Assert.AreEqual(warnings[1], warnings[2]); + Assert.AreEqual("ObjectStore: key 'trade_log_ai:cloud.csv' is not supported and its sanitized name 'trade_log_ai_cloud.csv' " + + "already holds another object: the save is rejected. Delete 'trade_log_ai_cloud.csv' or save to it directly to replace it", warnings[3]); + Assert.AreEqual("existing", store.Read("trade_log_ai_cloud.csv")); + + // once the name is free the key is saved there and updated silently + Assert.IsTrue(store.Delete("trade_log_ai_cloud.csv")); + Assert.IsTrue(store.Save("trade_log_ai&cloud.csv", "first")); Assert.IsTrue(store.Save("trade_log_ai&cloud.csv", "second")); + Assert.AreEqual(4, warnings.Count, string.Join("\n", warnings)); + Assert.AreEqual("second", store.Read("trade_log_ai_cloud.csv")); + Assert.AreEqual(1, store.Keys.Count(key => key.StartsWith("trade_log_ai"))); + + Assert.IsTrue(store.Delete("trade_log_ai&cloud.csv")); + Assert.IsFalse(store.ContainsKey("trade_log_ai_cloud.csv")); + } + + [Test] + public void ASaveToTheSanitizedNameFromAnotherKeyEndsTheSilentUpdatesOfAnUnsupportedKey() + { + var warnings = new List(); + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + + Assert.IsTrue(store.Save("p&q.csv", "A1")); + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); + + // another unsupported key sanitizing to the same name is rejected, and the first key keeps updating silently + Assert.IsFalse(store.Save("p:q.csv", "B1")); + Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); + Assert.That(warnings[1], Does.StartWith("ObjectStore: key 'p:q.csv' is not supported and its sanitized name 'p_q.csv' already holds another object")); + Assert.AreEqual("A1", store.Read("p_q.csv")); + Assert.IsTrue(store.Save("p&q.csv", "A2")); Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); + Assert.AreEqual("A2", store.Read("p_q.csv")); - // a different unsupported key sanitizing to the same object warns again - Assert.IsTrue(store.Save("trade_log_ai:cloud.csv", "third")); + // a save through the valid name takes the object over: the unsupported key is now rejected + Assert.IsTrue(store.Save("p_q.csv", "C1")); + Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); + Assert.IsFalse(store.Save("p&q.csv", "A3")); Assert.AreEqual(3, warnings.Count, string.Join("\n", warnings)); - Assert.AreEqual("ObjectStore: key 'trade_log_ai:cloud.csv' is not supported and is saved as 'trade_log_ai_cloud.csv', " + - "overwriting the existing object 'trade_log_ai_cloud.csv'", warnings[2]); - Assert.AreEqual("third", store.Read("trade_log_ai_cloud.csv")); - Assert.AreEqual(1, store.Keys.Count(key => key.StartsWith("trade_log_ai"))); + Assert.That(warnings[2], Does.StartWith("ObjectStore: key 'p&q.csv' is not supported and its sanitized name 'p_q.csv' already holds another object")); + Assert.AreEqual("C1", store.Read("p_q.csv")); - store.Delete("trade_log_ai_cloud.csv"); + Assert.IsTrue(store.Delete("p_q.csv")); + Assert.IsTrue(store.Save("p&q.csv", "A3")); + Assert.AreEqual(3, warnings.Count, string.Join("\n", warnings)); + Assert.AreEqual("A3", store.Read("p&q.csv")); + Assert.IsTrue(store.Delete("p&q.csv")); + Assert.IsFalse(store.ContainsKey("p_q.csv")); } [Test] From 3c973ad39bc0615bcb1eb8fd0a6cd022aabd8841 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 3 Nov 2026 18:12:33 -0400 Subject: [PATCH 12/14] Throw when a sanitized save would replace an object written from another key Nothing records who wrote the object under a sanitized key: a previous run, another algorithm or an upload. A save through the unsupported key throws instead of returning false, so the run fails visibly rather than losing its output to a debug message. --- Common/Storage/ObjectStore.cs | 30 +++---- Tests/Common/Storage/LocalObjectStoreTests.cs | 83 ++++++++++++------- 2 files changed, 67 insertions(+), 46 deletions(-) diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index 40fc01ae379c..3bebcf80d787 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -35,8 +35,9 @@ namespace QuantConnect.Storage /// format (english letters, numbers, spaces and the characters '/', '\', '_', '#', '-', '$' and '=', plus at /// most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv') is saved /// under its sanitized version and found by either name, with a warning the first time it happens. A save whose - /// sanitized name already holds an object this store did not write from that key is rejected with a warning. An object - /// already stored under an unsupported name is read as is; a save to that name goes to the sanitized name, warning every time + /// sanitized name already holds an object this store did not write from that key throws, since nothing records who + /// wrote that object. An object already stored under an unsupported name is read as is; a save to that name goes + /// to the sanitized name, warning every time /// public class ObjectStore : IObjectStore { @@ -132,7 +133,8 @@ public byte[] ReadBytes(string path) /// True if the save operation was successful public bool SaveBytes(string path, byte[] contents) { - if (!TryGetKeyToSave(path, out var key) || !_store.SaveBytes(key, contents)) + var key = KeyToSave(path); + if (!_store.SaveBytes(key, contents)) { return false; } @@ -373,32 +375,30 @@ internal static void ValidateKey(string key) } /// - /// The key a save goes to: the given one when it follows the key format, else its sanitized version, unless that - /// already holds an object this store did not last write from the given key. Null and empty keys are left for the store to reject + /// The key a save goes to: the given one when it follows the key format, else its sanitized version. Throws when + /// that already holds an object this store did not last write from the given key. Null and empty keys are left for the store to reject /// - private bool TryGetKeyToSave(string path, out string key) + private string KeyToSave(string path) { - key = path; if (string.IsNullOrEmpty(path) || IsSupportedKey(path)) { - return true; + return path; } - key = SanitizeKey(path); + var key = SanitizeKey(path); WarnOnce(path, key); if (!_savedKeys.ContainsKey(path) && _store.ContainsKey(key)) { - // the rejection used to prevent this: silently replacing an object saved under another key - _warn($"ObjectStore: key '{path}' is not supported and its sanitized name '{key}' already holds another object:" + - $" the save is rejected. Delete '{key}' or save to it directly to replace it"); - return false; + // nothing records who wrote that object, a previous run, another algorithm or an upload: it is never replaced through another key + throw new ArgumentException($"ObjectStore: key '{path}' is not supported and its sanitized name '{key}' already holds" + + $" another object. Delete '{key}' or save to it directly to replace it"); } if (_store.ContainsKey(path)) { // the object under the name as given stays: reads of it from another store still return it _warn($"ObjectStore: key '{path}' is not supported: the object stored under that name is left as is, this save goes to '{key}'"); } - return true; + return key; } /// @@ -444,7 +444,7 @@ private void WarnOnce(string path, string key) if (Interlocked.Exchange(ref _warned, 1) == 0) { _warn($"ObjectStore: key '{path}' is not supported: {GetKeyIssues(path)}. Using '{key}' instead. Any other unsupported" + - " key is sanitized the same way; a save is rejected, with a warning, when its sanitized name already holds another object"); + " key is sanitized the same way; a save throws when its sanitized name already holds an object not written from that key in this run"); } } diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index 6d78d0049a7d..6a83865a1d3c 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -630,7 +630,7 @@ public void UnsupportedKeysAreWarnedAboutOnceStatingTheIssueAndTheKeyUsed() "Keys may only contain english letters, numbers, spaces and the characters '/', '\\', '_', '#', '-', '$' and '=', " + "plus at most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv'. " + "Using 'trade log_ AI _ Cloud.csv' instead. Any other unsupported key is sanitized the same way; " + - "a save is rejected, with a warning, when its sanitized name already holds another object", warnings[0]); + "a save throws when its sanitized name already holds an object not written from that key in this run", warnings[0]); // the next unsupported key is sanitized silently Assert.IsFalse(store.ContainsKey("trade log: AI.v2.csv")); @@ -743,18 +743,17 @@ public void SavingToAnExistingUnsupportedKeyGoesToTheSanitizedNameWithAWarningEv Assert.AreEqual("old", store.Read("model.tar.gz")); Assert.IsEmpty(warnings); - Assert.IsFalse(store.Save("model.tar.gz", "third")); - - Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); + var exception = Assert.Throws(() => store.Save("model.tar.gz", "third")); Assert.AreEqual("ObjectStore: key 'model.tar.gz' is not supported and its sanitized name 'model_tar.gz' already holds" + - " another object: the save is rejected. Delete 'model_tar.gz' or save to it directly to replace it", warnings[1]); + " another object. Delete 'model_tar.gz' or save to it directly to replace it", exception.Message); + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); Assert.AreEqual("newer", store.Read("model_tar.gz")); // once the sanitized name is free the save goes there, warning about the original every time Assert.IsTrue(store.Delete("model_tar.gz")); Assert.IsTrue(store.Save("model.tar.gz", "third")); - Assert.AreEqual(3, warnings.Count, string.Join("\n", warnings)); - Assert.That(warnings[2], Does.Contain("the object stored under that name is left as is, this save goes to 'model_tar.gz'")); + Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); + Assert.That(warnings[1], Does.Contain("the object stored under that name is left as is, this save goes to 'model_tar.gz'")); Assert.AreEqual("third", store.Read("model.tar.gz")); Assert.AreEqual("third", store.Read("model_tar.gz")); @@ -826,7 +825,7 @@ public void CustomDataIsReadFromAnExistingUnsupportedKey() } [Test] - public void SavingAnUnsupportedKeyWhoseSanitizedNameHoldsAnotherObjectIsRejectedWithAWarningEveryTime() + public void SavingAnUnsupportedKeyWhoseSanitizedNameHoldsAnotherObjectThrowsEveryTime() { var warnings = new List(); using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); @@ -834,28 +833,27 @@ public void SavingAnUnsupportedKeyWhoseSanitizedNameHoldsAnotherObjectIsRejected Assert.IsTrue(store.Save("trade_log_ai_cloud.csv", "existing")); Assert.IsEmpty(warnings); - // sanitizes to the existing object's key: the general warning, then the rejection - Assert.IsFalse(store.Save("trade_log_ai&cloud.csv", "first")); - Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); - Assert.That(warnings[0], Does.StartWith("ObjectStore: key 'trade_log_ai&cloud.csv' is not supported: unsupported character '&'.")); + // sanitizes to the existing object's key: the general warning, then the exception + var exception = Assert.Throws(() => store.Save("trade_log_ai&cloud.csv", "first")); Assert.AreEqual("ObjectStore: key 'trade_log_ai&cloud.csv' is not supported and its sanitized name 'trade_log_ai_cloud.csv' " + - "already holds another object: the save is rejected. Delete 'trade_log_ai_cloud.csv' or save to it directly to replace it", warnings[1]); + "already holds another object. Delete 'trade_log_ai_cloud.csv' or save to it directly to replace it", exception.Message); + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); + Assert.That(warnings[0], Does.StartWith("ObjectStore: key 'trade_log_ai&cloud.csv' is not supported: unsupported character '&'.")); Assert.AreEqual("existing", store.Read("trade_log_ai_cloud.csv")); - // every such save is rejected with the warning, from this key or another one sanitizing to the same name - Assert.IsFalse(store.Save("trade_log_ai&cloud.csv", "first")); - Assert.IsFalse(store.Save("trade_log_ai:cloud.csv", "second")); - Assert.AreEqual(4, warnings.Count, string.Join("\n", warnings)); - Assert.AreEqual(warnings[1], warnings[2]); + // every such save throws, from this key or another one sanitizing to the same name + Assert.Throws(() => store.Save("trade_log_ai&cloud.csv", "first")); + exception = Assert.Throws(() => store.Save("trade_log_ai:cloud.csv", "second")); Assert.AreEqual("ObjectStore: key 'trade_log_ai:cloud.csv' is not supported and its sanitized name 'trade_log_ai_cloud.csv' " + - "already holds another object: the save is rejected. Delete 'trade_log_ai_cloud.csv' or save to it directly to replace it", warnings[3]); + "already holds another object. Delete 'trade_log_ai_cloud.csv' or save to it directly to replace it", exception.Message); + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); Assert.AreEqual("existing", store.Read("trade_log_ai_cloud.csv")); // once the name is free the key is saved there and updated silently Assert.IsTrue(store.Delete("trade_log_ai_cloud.csv")); Assert.IsTrue(store.Save("trade_log_ai&cloud.csv", "first")); Assert.IsTrue(store.Save("trade_log_ai&cloud.csv", "second")); - Assert.AreEqual(4, warnings.Count, string.Join("\n", warnings)); + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); Assert.AreEqual("second", store.Read("trade_log_ai_cloud.csv")); Assert.AreEqual(1, store.Keys.Count(key => key.StartsWith("trade_log_ai"))); @@ -863,6 +861,32 @@ public void SavingAnUnsupportedKeyWhoseSanitizedNameHoldsAnotherObjectIsRejected Assert.IsFalse(store.ContainsKey("trade_log_ai_cloud.csv")); } + [Test] + public void ANewRunThrowsOnSavingAnUnsupportedKeyAPreviousRunSanitized() + { + // nothing records that a previous run wrote the sanitized object, so no later run replaces it through the unsupported key + using (var firstRun = new ObjectStore(new TestLocalObjectStore(), _ => { })) + { + firstRun.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.IsTrue(firstRun.Save("trade log: AI & Cloud.csv", "run1")); + } + + using (var secondRun = new ObjectStore(new TestLocalObjectStore(), _ => { })) + { + secondRun.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + var exception = Assert.Throws(() => secondRun.Save("trade log: AI & Cloud.csv", "run2")); + Assert.AreEqual("ObjectStore: key 'trade log: AI & Cloud.csv' is not supported and its sanitized name 'trade log_ AI _ Cloud.csv'" + + " already holds another object. Delete 'trade log_ AI _ Cloud.csv' or save to it directly to replace it", exception.Message); + Assert.AreEqual("run1", secondRun.Read("trade log: AI & Cloud.csv")); + + // the exact sanitized key is a supported key: the user replaces the object through it + Assert.IsTrue(secondRun.Save("trade log_ AI _ Cloud.csv", "run2")); + Assert.AreEqual("run2", secondRun.Read("trade log: AI & Cloud.csv")); + Assert.Throws(() => secondRun.Save("trade log: AI & Cloud.csv", "run2 again")); + Assert.IsTrue(secondRun.Delete("trade log_ AI _ Cloud.csv")); + } + } + [Test] public void ASaveToTheSanitizedNameFromAnotherKeyEndsTheSilentUpdatesOfAnUnsupportedKey() { @@ -873,26 +897,23 @@ public void ASaveToTheSanitizedNameFromAnotherKeyEndsTheSilentUpdatesOfAnUnsuppo Assert.IsTrue(store.Save("p&q.csv", "A1")); Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); - // another unsupported key sanitizing to the same name is rejected, and the first key keeps updating silently - Assert.IsFalse(store.Save("p:q.csv", "B1")); - Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); - Assert.That(warnings[1], Does.StartWith("ObjectStore: key 'p:q.csv' is not supported and its sanitized name 'p_q.csv' already holds another object")); + // another unsupported key sanitizing to the same name throws, and the first key keeps updating silently + var exception = Assert.Throws(() => store.Save("p:q.csv", "B1")); + Assert.That(exception.Message, Does.StartWith("ObjectStore: key 'p:q.csv' is not supported and its sanitized name 'p_q.csv' already holds another object")); Assert.AreEqual("A1", store.Read("p_q.csv")); Assert.IsTrue(store.Save("p&q.csv", "A2")); - Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); Assert.AreEqual("A2", store.Read("p_q.csv")); - // a save through the valid name takes the object over: the unsupported key is now rejected + // a save through the valid name takes the object over: the unsupported key now throws Assert.IsTrue(store.Save("p_q.csv", "C1")); - Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); - Assert.IsFalse(store.Save("p&q.csv", "A3")); - Assert.AreEqual(3, warnings.Count, string.Join("\n", warnings)); - Assert.That(warnings[2], Does.StartWith("ObjectStore: key 'p&q.csv' is not supported and its sanitized name 'p_q.csv' already holds another object")); + exception = Assert.Throws(() => store.Save("p&q.csv", "A3")); + Assert.That(exception.Message, Does.StartWith("ObjectStore: key 'p&q.csv' is not supported and its sanitized name 'p_q.csv' already holds another object")); Assert.AreEqual("C1", store.Read("p_q.csv")); Assert.IsTrue(store.Delete("p_q.csv")); Assert.IsTrue(store.Save("p&q.csv", "A3")); - Assert.AreEqual(3, warnings.Count, string.Join("\n", warnings)); + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); Assert.AreEqual("A3", store.Read("p&q.csv")); Assert.IsTrue(store.Delete("p&q.csv")); Assert.IsFalse(store.ContainsKey("p_q.csv")); From 474b1016ea4df5824041e6dabcaef0e67aa595c0 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 4 Nov 2026 13:37:29 -0400 Subject: [PATCH 13/14] Move the key sanitization rules to their own PR The ObjectStore keeps the Python helpers, the SaveText/ReadText aliases and the keys listed in the missing-key error. Sanitizing unsupported keys, the collisions that throw, the warnings and the LocalObjectStore backstop move to a separate branch off master. --- ...bjectStoreErgonomicsRegressionAlgorithm.cs | 31 +- ...bjectStoreErgonomicsRegressionAlgorithm.py | 26 +- Algorithm/QCAlgorithm.cs | 2 +- .../Python/Wrappers/AlgorithmPythonWrapper.cs | 2 +- Common/Storage/ObjectStore.cs | 270 +---------- Common/Storage/PythonObjectStore.cs | 11 +- Engine/Storage/LocalObjectStore.cs | 12 +- Research/QuantBook.cs | 1 - Tests/Common/Storage/LocalObjectStoreTests.cs | 421 +----------------- Tests/Python/AlgorithmPythonWrapperTests.cs | 13 +- 10 files changed, 70 insertions(+), 719 deletions(-) diff --git a/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs b/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs index a290e329de2a..ba40d977d328 100644 --- a/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs +++ b/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs @@ -20,10 +20,10 @@ namespace QuantConnect.Algorithm.CSharp { /// - /// Regression algorithm asserting the ObjectStore ergonomics: an unsupported key is sanitized and used on - /// every save and read, and - /// are aliases of Save and Read, and reading a missing key lists the available keys in the error. The Python - /// version of this algorithm additionally covers the tolerant save_json/read_json and save_dataframe/read_dataframe helpers + /// Regression algorithm asserting the ObjectStore ergonomics: and + /// are aliases of Save and Read, and reading a missing key lists the + /// available keys in the error. The Python version of this algorithm additionally covers the tolerant + /// save_json/read_json and save_dataframe/read_dataframe helpers /// public class ObjectStoreErgonomicsRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition { @@ -45,22 +45,6 @@ public override void Initialize() /// public override void OnEndOfAlgorithm() { - // an unsupported key, e.g. built from a user-facing name, is sanitized and used on every save and read - const string invalidKey = "trade_log_ai_hardware_&_cloud.csv"; - const string sanitizedKey = "trade_log_ai_hardware___cloud.csv"; - if (!ObjectStore.Save(invalidKey, "a,b\n1,2")) - { - throw new RegressionTestException("Expected the unsupported key to be sanitized and saved"); - } - if (!ObjectStore.Keys.Contains(sanitizedKey) || ObjectStore.Keys.Contains(invalidKey)) - { - throw new RegressionTestException($"Expected the object to be stored under '{sanitizedKey}', keys: {string.Join(", ", ObjectStore.Keys)}"); - } - if (!ObjectStore.ContainsKey(invalidKey) || ObjectStore.Read(invalidKey) != "a,b\n1,2" || ObjectStore.Read(sanitizedKey) != "a,b\n1,2") - { - throw new RegressionTestException("Expected the object to be read through both the original and the sanitized key"); - } - // SaveText and ReadText are aliases of Save and Read if (!ObjectStore.SaveText("ergonomics_report.txt", "The strategy went up") || ObjectStore.Read("ergonomics_report.txt") != "The strategy went up" || @@ -79,17 +63,12 @@ public override void OnEndOfAlgorithm() { errorMessage = exception.Message; } - if (!errorMessage.Contains("Keys: [") || !errorMessage.Contains($"'{sanitizedKey}'")) + if (!errorMessage.Contains("Keys: [") || !errorMessage.Contains("'ergonomics_report.txt'")) { throw new RegressionTestException($"Expected the available keys in the missing key error, got: '{errorMessage}'"); } - ObjectStore.Delete(invalidKey); ObjectStore.Delete("ergonomics_report.txt"); - if (ObjectStore.ContainsKey(sanitizedKey)) - { - throw new RegressionTestException("Expected the delete through the original key to remove the sanitized one"); - } } /// diff --git a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py index a0ffad2e485a..5a5b73147dc5 100644 --- a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py +++ b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py @@ -16,11 +16,9 @@ from decimal import Decimal ### -### Regression algorithm asserting the ObjectStore ergonomics: an unsupported key is sanitized and used on -### every save and read, save_text and read_text are aliases of save and read, reading a missing key lists the -### available keys in the error, and -### the tolerant save_json/read_json and save_dataframe/read_dataframe helpers handle datetime/date/Decimal/Symbol -### and data frames out of the box. +### Regression algorithm asserting the ObjectStore ergonomics: save_text and read_text are aliases of save and read, +### reading a missing key lists the available keys in the error, and the tolerant save_json/read_json and +### save_dataframe/read_dataframe helpers handle datetime/date/Decimal/Symbol and data frames out of the box. ### class ObjectStoreErgonomicsRegressionAlgorithm(QCAlgorithm): @@ -33,18 +31,6 @@ def initialize(self): self.spy = self.add_equity("SPY", Resolution.DAILY).symbol def on_end_of_algorithm(self): - # an unsupported key, e.g. built from a user-facing name, is sanitized and used on every save and read - invalid_key = "trade_log_ai_hardware_&_cloud.csv" - sanitized_key = "trade_log_ai_hardware___cloud.csv" - if not self.object_store.save(invalid_key, "a,b\n1,2"): - raise AssertionError("Expected the unsupported key to be sanitized and saved") - keys = list(self.object_store.keys) - if sanitized_key not in keys or invalid_key in keys: - raise AssertionError(f"Expected the object to be stored under '{sanitized_key}', keys: {keys}") - if not self.object_store.contains_key(invalid_key) or self.object_store.read(invalid_key) != "a,b\n1,2" \ - or self.object_store.read(sanitized_key) != "a,b\n1,2": - raise AssertionError("Expected the object to be read through both the original and the sanitized key") - # save_json tolerates datetime/date/Decimal/numpy/NaN values, .NET objects and non-string dictionary keys trade_log = { "time": datetime(2013, 10, 11, 16, 0, 0), @@ -116,10 +102,8 @@ def on_end_of_algorithm(self): self.object_store.read_bytes("ergonomics/missing.json") except BaseException as e: error = str(e) - if error is None or "Keys: [" not in error or f"'{sanitized_key}'" not in error: + if error is None or "Keys: [" not in error or "'ergonomics/report.txt'" not in error: raise AssertionError(f"Expected the available keys in the missing key error, got: '{error}'") - for key in [invalid_key, "ergonomics/trade_log.json", "ergonomics/history.csv", "ergonomics/report.txt"]: + for key in ["ergonomics/trade_log.json", "ergonomics/history.csv", "ergonomics/report.txt"]: self.object_store.delete(key) - if self.object_store.contains_key(sanitized_key): - raise AssertionError("Expected the delete through the original key to remove the sanitized one") diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index 315d70a21af1..520867b545e7 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -3309,7 +3309,7 @@ public void SetApi(IApi api) public void SetObjectStore(IObjectStore objectStore) { // an already wrapped store, like the Python one, is used as is - ObjectStore = objectStore as ObjectStore ?? new ObjectStore(objectStore, Debug); + ObjectStore = objectStore as ObjectStore ?? new ObjectStore(objectStore); } /// diff --git a/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs b/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs index 526906505687..2fafbc4f675c 100644 --- a/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs +++ b/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs @@ -1231,7 +1231,7 @@ public void SetCurrentSlice(Slice slice) /// Sets the object store /// /// The object store - public void SetObjectStore(IObjectStore objectStore) => _baseAlgorithm.SetObjectStore(new PythonObjectStore(objectStore, _baseAlgorithm.Debug)); + public void SetObjectStore(IObjectStore objectStore) => _baseAlgorithm.SetObjectStore(new PythonObjectStore(objectStore)); /// /// Determines if the Symbol is shortable at the brokerage diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index 3bebcf80d787..a610c1d00752 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -15,43 +15,21 @@ using System; using System.Collections; -using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Text; -using System.Threading; -using System.Text.RegularExpressions; using System.Xml.Serialization; using Newtonsoft.Json; using QuantConnect.Interfaces; -using QuantConnect.Logging; using QuantConnect.Packets; namespace QuantConnect.Storage { /// - /// Helper class for easier access to methods. A key that does not follow the key - /// format (english letters, numbers, spaces and the characters '/', '\', '_', '#', '-', '$' and '=', plus at - /// most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv') is saved - /// under its sanitized version and found by either name, with a warning the first time it happens. A save whose - /// sanitized name already holds an object this store did not write from that key throws, since nothing records who - /// wrote that object. An object already stored under an unsupported name is read as is; a save to that name goes - /// to the sanitized name, warning every time + /// Helper class for easier access to methods /// public class ObjectStore : IObjectStore { - // The single source of the key charset/extension rules, also enforced by the LocalObjectStore implementation - private static readonly Regex SupportedKeyRegex = new(@"^\.?[a-zA-Z0-9\\/_#\-\$= ]+\.?[a-zA-Z0-9]*$", RegexOptions.Compiled); - private static readonly Regex UnsupportedKeyCharactersRegex = new(@"[^a-zA-Z0-9\\/_#\-\$= .]", RegexOptions.Compiled); - private static readonly Regex KeyExtensionRegex = new(@"^[a-zA-Z0-9]+$", RegexOptions.Compiled); - - private readonly IObjectStore _store; - private readonly Action _warn; - private int _warned; - // the unsupported keys this store last wrote each sanitized name from: their saves are updates, and reads follow them - private readonly ConcurrentDictionary _savedKeys = new(); - /// /// Gets the maximum storage limit in bytes /// @@ -71,15 +49,15 @@ public event EventHandler ErrorRaised remove { _store.ErrorRaised -= value; } } + private readonly IObjectStore _store; + /// /// Initializes a new instance of the class /// /// The instance to wrap - /// Receives the warnings about sanitized keys, the engine log by default - public ObjectStore(IObjectStore store, Action warn = null) + public ObjectStore(IObjectStore store) { _store = store; - _warn = warn ?? (message => Log.Trace(message)); } /// @@ -112,7 +90,7 @@ public void Initialize(int userId, int projectId, string userToken, Controls con /// True if the key was found public bool ContainsKey(string path) { - return _store.ContainsKey(KeyToRead(path)); + return _store.ContainsKey(path); } /// @@ -122,28 +100,18 @@ public bool ContainsKey(string path) /// A byte array containing the data public byte[] ReadBytes(string path) { - return _store.ReadBytes(KeyToRead(path)); + return _store.ReadBytes(path); } /// /// Saves the object data for the specified path /// - /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' + /// The object path /// The object data /// True if the save operation was successful public bool SaveBytes(string path, byte[] contents) { - var key = KeyToSave(path); - if (!_store.SaveBytes(key, contents)) - { - return false; - } - ForgetSavesTo(key, exceptFrom: path); - if (key != path) - { - _savedKeys[path] = key; - } - return true; + return _store.SaveBytes(path, contents); } /// @@ -153,13 +121,7 @@ public bool SaveBytes(string path, byte[] contents) /// True if the delete operation was successful public bool Delete(string path) { - var key = KeyToRead(path); - if (!_store.Delete(key)) - { - return false; - } - ForgetSavesTo(key); - return true; + return _store.Delete(path); } /// @@ -169,7 +131,7 @@ public bool Delete(string path) /// The path for the file public string GetFilePath(string path) { - return _store.GetFilePath(KeyToRead(path)); + return _store.GetFilePath(path); } /// @@ -182,7 +144,7 @@ public string Read(string path, Encoding encoding = null) { encoding = encoding ?? Encoding.UTF8; - var data = ReadBytes(path); + var data = _store.ReadBytes(path); return data != null ? encoding.GetString(data) : null; } @@ -246,7 +208,7 @@ public T ReadXml(string path, Encoding encoding = null) /// Saves the data from a local file path associated with the specified path /// /// If the file does not exist it will throw an exception - /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' + /// The object path /// True if the object was saved successfully public bool Save(string path) { @@ -258,39 +220,41 @@ public bool Save(string path) } var bytes = File.ReadAllBytes(filePath); - return SaveBytes(path, bytes); + return _store.SaveBytes(path, bytes); } /// /// Saves the object data in text format for the specified path /// - /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' + /// The object path /// The string object to be saved /// The string encoding used, by default /// True if the object was saved successfully public bool Save(string path, string text, Encoding encoding = null) { encoding ??= Encoding.UTF8; - return SaveBytes(path, encoding.GetBytes(text)); + return _store.SaveBytes(path, encoding.GetBytes(text)); } /// /// Saves the object data in text format for the specified path /// - /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' + /// The object path /// The string object to be saved /// The string encoding used /// True if the object was saved successfully public bool SaveString(string path, string text, Encoding encoding = null) { - return Save(path, text, encoding); + encoding = encoding ?? Encoding.UTF8; + + return _store.SaveBytes(path, encoding.GetBytes(text)); } /// /// Saves the object data in text format for the specified path. /// Alias of /// - /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' + /// The object path /// The string object to be saved /// The string encoding used, by default /// True if the object was saved successfully @@ -302,7 +266,7 @@ public bool SaveText(string path, string text, Encoding encoding = null) /// /// Saves the object data in JSON format for the specified path /// - /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' + /// The object path /// The object to be saved /// The string encoding used /// The settings used by the JSON serializer @@ -318,7 +282,7 @@ public bool SaveJson(string path, T obj, Encoding encoding = null, JsonSerial /// /// Saves the object data in XML format for the specified path /// - /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' + /// The object path /// The object to be saved /// The string encoding used /// True if the object was saved successfully @@ -358,195 +322,5 @@ public void Dispose() { _store.Dispose(); } - - /// - /// Throws if the given key does not follow the key format, stating every problem found and the rules. - /// The last line of defense of the store implementations, as this class only hands them supported keys - /// - /// The object store key to validate - /// The key is not supported - internal static void ValidateKey(string key) - { - var issues = GetKeyIssues(key); - if (issues != null) - { - throw new ArgumentException($"ObjectStore: path is not supported: '{key}': {issues}"); - } - } - - /// - /// The key a save goes to: the given one when it follows the key format, else its sanitized version. Throws when - /// that already holds an object this store did not last write from the given key. Null and empty keys are left for the store to reject - /// - private string KeyToSave(string path) - { - if (string.IsNullOrEmpty(path) || IsSupportedKey(path)) - { - return path; - } - - var key = SanitizeKey(path); - WarnOnce(path, key); - if (!_savedKeys.ContainsKey(path) && _store.ContainsKey(key)) - { - // nothing records who wrote that object, a previous run, another algorithm or an upload: it is never replaced through another key - throw new ArgumentException($"ObjectStore: key '{path}' is not supported and its sanitized name '{key}' already holds" + - $" another object. Delete '{key}' or save to it directly to replace it"); - } - if (_store.ContainsKey(path)) - { - // the object under the name as given stays: reads of it from another store still return it - _warn($"ObjectStore: key '{path}' is not supported: the object stored under that name is left as is, this save goes to '{key}'"); - } - return key; - } - - /// - /// Forgets the unsupported keys whose object under the given key is gone or replaced from another key - /// - private void ForgetSavesTo(string key, string exceptFrom = null) - { - foreach (var entry in _savedKeys) - { - if (entry.Value == key && entry.Key != exceptFrom) - { - _savedKeys.TryRemove(entry.Key, out _); - } - } - } - - /// - /// The key the other operations use: where this store saved it while that object exists, else the given one when - /// it follows the key format or the store already has it, else its sanitized version, so an object is found by the key it was saved with - /// - private string KeyToRead(string path) - { - if (string.IsNullOrEmpty(path)) - { - return path; - } - if (_savedKeys.TryGetValue(path, out var key) && _store.ContainsKey(key)) - { - return key; - } - if (IsSupportedKey(path) || _store.ContainsKey(path)) - { - return path; - } - - key = SanitizeKey(path); - WarnOnce(path, key); - return key; - } - - private void WarnOnce(string path, string key) - { - if (Interlocked.Exchange(ref _warned, 1) == 0) - { - _warn($"ObjectStore: key '{path}' is not supported: {GetKeyIssues(path)}. Using '{key}' instead. Any other unsupported" + - " key is sanitized the same way; a save throws when its sanitized name already holds an object not written from that key in this run"); - } - } - - private static bool IsSupportedKey(string key) - { - return !string.IsNullOrEmpty(key) && SupportedKeyRegex.IsMatch(key) - // just in case - && key.Count(c => c == '/') <= 100 && key.Count(c => c == '\\') <= 100; - } - - /// - /// Describes why a key does not follow the key format: the problems found and the rules. Null for a supported key - /// - private static string GetKeyIssues(string key) - { - if (IsSupportedKey(key)) - { - return null; - } - return string.Join("; ", FindKeyIssues(key)) + - ". Keys may only contain english letters, numbers, spaces and the characters" + - " '/', '\\', '_', '#', '-', '$' and '=', plus at most one '.' followed by a letters-and-numbers-only extension," + - " e.g. 'folder/trade_log-2024.csv'"; - } - - /// - /// Lists why a key fails , one entry per problem found - /// - private static List FindKeyIssues(string key) - { - var issues = new List(); - if (string.IsNullOrEmpty(key)) - { - issues.Add("the key is empty"); - return issues; - } - - var unsupported = UnsupportedKeyCharactersRegex.Matches(key).Select(match => $"'{match.Value}'").Distinct().ToList(); - if (unsupported.Count > 0) - { - issues.Add($"unsupported character{(unsupported.Count > 1 ? "s" : "")} {string.Join(", ", unsupported)}"); - } - - // a single leading '.' is allowed, every other '.' must introduce the final extension - var name = key.TrimStart('.'); - var dots = name.Count(c => c == '.'); - if (dots > 1) - { - issues.Add($"it has {dots} '.' but at most one '.extension' is allowed"); - } - else if (dots == 1) - { - var extension = name[(name.IndexOf('.', StringComparison.Ordinal) + 1)..]; - if (extension.Length > 0 && !KeyExtensionRegex.IsMatch(extension)) - { - issues.Add($"the extension '{extension}' may only contain letters and numbers"); - } - } - if (name.Length == 0) - { - issues.Add("it has no name before the '.'"); - } - else if (key.StartsWith("..", StringComparison.Ordinal)) - { - issues.Add("it starts with more than one '.'"); - } - - if (key.Count(c => c == '/') > 100 || key.Count(c => c == '\\') > 100) - { - issues.Add("it has more than 100 '/' or '\\' separators"); - } - - if (issues.Count == 0) - { - issues.Add("it does not follow the key format"); - } - return issues; - } - - /// - /// Converts an unsupported key into a supported one by replacing every unsupported character with '_', - /// keeping at most the final '.extension': 'trade log: AI & Cloud.csv' becomes 'trade log_ AI _ Cloud.csv'. - /// The mapping is lossy, distinct keys can sanitize to the same one. Throws when the key cannot be fixed - /// - private static string SanitizeKey(string key) - { - var sanitized = UnsupportedKeyCharactersRegex.Replace(key, "_"); - - // a single trailing '.extension' of letters and numbers is allowed: keep the last dot if it - // introduces a valid extension and replace every other dot - var lastDot = sanitized.LastIndexOf('.'); - if (lastDot > 0 && KeyExtensionRegex.IsMatch(sanitized.AsSpan(lastDot + 1))) - { - sanitized = string.Concat(sanitized[..lastDot].Replace('.', '_'), sanitized.AsSpan(lastDot)); - } - else if (lastDot >= 0) - { - sanitized = sanitized.Replace('.', '_'); - } - - ValidateKey(sanitized); - return sanitized; - } } } diff --git a/Common/Storage/PythonObjectStore.cs b/Common/Storage/PythonObjectStore.cs index 30c0a57ad8d3..a6792db05535 100644 --- a/Common/Storage/PythonObjectStore.cs +++ b/Common/Storage/PythonObjectStore.cs @@ -13,7 +13,6 @@ * limitations under the License. */ -using System; using Python.Runtime; using QuantConnect.Interfaces; @@ -35,10 +34,8 @@ public class PythonObjectStore : ObjectStore /// Initializes a new instance of the class /// /// The instance to wrap - /// Receives the warning logged the first time an unsupported key is sanitized, - /// the engine log by default - public PythonObjectStore(IObjectStore store, Action warn = null) - : base(store, warn) + public PythonObjectStore(IObjectStore store) + : base(store) { } @@ -88,7 +85,7 @@ public PyObject ReadDataframe(string path) /// , and so is any other type that would not round trip. A top-level Python /// str, int, float or bool is not accepted, use for text /// - /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' + /// The object path /// The Python object to be saved /// True if the object was saved successfully /// A TypeError naming the value that cannot be serialized @@ -108,7 +105,7 @@ public bool SaveJson(string path, PyObject obj) /// Saves the given pandas DataFrame (or Series) for the specified path, in JSON format if the /// path has a '.json' extension and as CSV otherwise /// - /// The object key: english letters, numbers, spaces, '/', '\', '_', '#', '-', '$', '=' and at most one '.extension', e.g. 'folder/trade_log-2024.csv'. Other characters are replaced by '_' + /// The object path /// The pandas DataFrame or Series to be saved /// True if the object was saved successfully public bool SaveDataframe(string path, PyObject dataFrame) diff --git a/Engine/Storage/LocalObjectStore.cs b/Engine/Storage/LocalObjectStore.cs index 679ef52f414c..bbe6d7ccaf81 100644 --- a/Engine/Storage/LocalObjectStore.cs +++ b/Engine/Storage/LocalObjectStore.cs @@ -19,6 +19,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.RegularExpressions; using System.Threading; using QuantConnect.Configuration; using QuantConnect.Interfaces; @@ -111,6 +112,7 @@ private bool IsDirty } private Timer _persistenceTimer; + private Regex _pathRegex = new(@"^\.?[a-zA-Z0-9\\/_#\-\$= ]+\.?[a-zA-Z0-9]*$", RegexOptions.Compiled); private readonly ConcurrentDictionary _storage = new(); private readonly object _persistLock = new object(); @@ -331,7 +333,15 @@ public bool SaveBytes(string path, byte[] contents) { throw new InvalidOperationException($"LocalObjectStore.SaveBytes(): {NoWritePermissionsError}"); } - ObjectStore.ValidateKey(path); + else if (!_pathRegex.IsMatch(path)) + { + throw new ArgumentException($"LocalObjectStore: path is not supported: '{path}'"); + } + else if (path.Count(c => c == '/') > 100 || path.Count(c => c == '\\') > 100) + { + // just in case + throw new ArgumentException($"LocalObjectStore: path is not supported: '{path}'"); + } // after we check the regex path = NormalizePath(path); diff --git a/Research/QuantBook.cs b/Research/QuantBook.cs index ad9678d43726..f53cab5681b8 100644 --- a/Research/QuantBook.cs +++ b/Research/QuantBook.cs @@ -166,7 +166,6 @@ public QuantBook() : base() StorageAccess = Config.GetValue("storage-permissions", new Packets.StoragePermissions()) }, AlgorithmMode.Research); - // key warnings go to the engine log, like every other QuantBook message: nothing drains the debug messages here SetObjectStore(_isPythonNotebook ? new PythonObjectStore(algorithmHandlers.ObjectStore) : new ObjectStore(algorithmHandlers.ObjectStore)); diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index 6a83865a1d3c..b1a42909d1ae 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -27,7 +27,6 @@ using System.Threading.Tasks; using System.Collections.Generic; using QuantConnect.Lean.Engine.Storage; -using QuantConnect.Lean.Engine.DataFeeds.Transport; using System.Reflection; using Python.Runtime; @@ -235,9 +234,9 @@ public void DeletePermissions([Values] bool read, [Values] bool write, [Values] } } - [TestCase("../prefix/", "__/prefix/")] - [TestCase("..\\prefix/", "__/prefix/")] - public void InvalidCustomPathsStore(string path, string sanitizedPath) + [TestCase("../prefix/")] + [TestCase("..\\prefix/")] + public void InvalidCustomPathsStore(string path) { #pragma warning disable CA2000 using (var store = new ObjectStore(new TestLocalObjectStore())) @@ -246,12 +245,7 @@ public void InvalidCustomPathsStore(string path, string sanitizedPath) store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); Assert.AreEqual(0, store.Count()); - // '..' cannot leave the storage root: it is sanitized away - Assert.IsTrue(store.SaveString($"{path}ILove", "Pizza")); - Assert.AreEqual(new[] { $"{sanitizedPath}ILove" }, store.Keys); - Assert.That(Path.GetFullPath(store.GetFilePath($"{path}ILove")), Does.StartWith(Path.GetFullPath(TestStorageRoot))); - - store.Delete($"{path}ILove"); + Assert.Throws(() => store.SaveString($"{path}ILove", "Pizza")); } } @@ -589,351 +583,6 @@ public void KeyNotFoundErrorTruncatesTheAvailableKeysListAtTen() } } - [TestCase("my_key/file.csv", "my_key/file.csv")] - [TestCase("trade_log_ai_hardware_&_cloud.csv", "trade_log_ai_hardware___cloud.csv")] - [TestCase("trade log: AI & Cloud.csv", "trade log_ AI _ Cloud.csv")] - [TestCase("file.tar.gz", "file_tar.gz")] - [TestCase("résumé.pdf", "r_sum_.pdf")] - [TestCase("a..", "a__")] - [TestCase("100%.json", "100_.json")] - [TestCase("..model.pkl", "__model.pkl")] - public void UnsupportedKeysAreSanitizedOnSaveAndFoundByTheKeyTheyWereSavedWith(string key, string sanitizedKey) - { - Assert.IsTrue(_store.Save(key, "abc")); - - Assert.IsTrue(_store.Keys.Contains(sanitizedKey), string.Join(", ", _store.Keys)); - Assert.AreEqual(key == sanitizedKey, _store.Keys.Contains(key)); - Assert.IsTrue(_store.ContainsKey(key)); - Assert.IsTrue(_store.ContainsKey(sanitizedKey)); - Assert.AreEqual("abc", _store.Read(key)); - Assert.AreEqual("abc", _store.Read(sanitizedKey)); - - Assert.IsTrue(_store.Delete(key)); - Assert.IsFalse(_store.ContainsKey(sanitizedKey)); - } - - [Test] - public void UnsupportedKeysAreWarnedAboutOnceStatingTheIssueAndTheKeyUsed() - { - var warnings = new List(); - using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); - store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); - - Assert.IsTrue(store.Save("trade log: AI & Cloud.csv", "abc")); - Assert.IsTrue(store.ContainsKey("trade log: AI & Cloud.csv")); - Assert.AreEqual("abc", store.Read("trade log: AI & Cloud.csv")); - Assert.IsTrue(store.Save("trade log: AI & Cloud.csv", "abcd")); - Assert.IsTrue(store.Save("supported.csv", "abc")); - - Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); - Assert.AreEqual("ObjectStore: key 'trade log: AI & Cloud.csv' is not supported: unsupported characters ':', '&'. " + - "Keys may only contain english letters, numbers, spaces and the characters '/', '\\', '_', '#', '-', '$' and '=', " + - "plus at most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv'. " + - "Using 'trade log_ AI _ Cloud.csv' instead. Any other unsupported key is sanitized the same way; " + - "a save throws when its sanitized name already holds an object not written from that key in this run", warnings[0]); - - // the next unsupported key is sanitized silently - Assert.IsFalse(store.ContainsKey("trade log: AI.v2.csv")); - Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); - - store.Delete("trade log: AI & Cloud.csv"); - store.Delete("supported.csv"); - } - - [TestCase("trade log: AI & Cloud.csv", "unsupported characters ':', '&'")] - [TestCase("résumé.pdf", "unsupported character 'é'")] - [TestCase("file.tar.gz", "it has 2 '.' but at most one '.extension' is allowed")] - [TestCase("file.name_v2", "the extension 'name_v2' may only contain letters and numbers")] - [TestCase("folder.v1/file", "the extension 'v1/file' may only contain letters and numbers")] - [TestCase("trade log: AI.v2.csv", "unsupported character ':'; it has 2 '.' but at most one '.extension' is allowed")] - [TestCase(".", "it has no name before the '.'")] - [TestCase("..model.pkl", "it starts with more than one '.'")] - public void UnsupportedKeyWarningStatesTheIssue(string key, string expectedIssue) - { - var warnings = new List(); - using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); - store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); - - Assert.IsFalse(store.ContainsKey(key)); - - Assert.AreEqual(1, warnings.Count); - Assert.That(warnings[0], Does.StartWith($"ObjectStore: key '{key}' is not supported: {expectedIssue}.")); - } - - [TestCase("")] - [TestCase(null)] - public void EmptyAndNullKeysAreRejected(string key) - { - var warnings = new List(); - using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); - store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); - - var error = Assert.Catch(() => store.Save(key, "abc")); - - Assert.IsEmpty(warnings); - if (key != null) - { - Assert.That(error.Message, Does.Contain("path is not supported: '': the key is empty.")); - } - } - - [TestCase("model.tar.gz")] - [TestCase("SPY (1).csv")] - [TestCase("BRK.B.csv")] - public void ExistingUnsupportedKeysAreUsedAsGiven(string key) - { - // an object put in the store by other means (uploaded, copied into the folder) under an unsupported name - var warnings = new List(); - using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); - store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); - var filePath = Path.Combine(TestStorageRoot, key); - File.WriteAllText(filePath, "abc"); - - Assert.IsTrue(store.ContainsKey(key)); - Assert.AreEqual("abc", store.Read(key)); - Assert.IsTrue(store.Keys.Contains(key), string.Join(", ", store.Keys)); - Assert.AreEqual(Path.GetFullPath(filePath), Path.GetFullPath(store.GetFilePath(key))); - Assert.IsEmpty(warnings, string.Join("\n", warnings)); - Assert.IsTrue(store.Delete(key)); - Assert.IsFalse(File.Exists(filePath)); - - // gone from the store, the key falls back to its sanitized name like any other unsupported key - Assert.IsFalse(store.ContainsKey(key)); - Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); - Assert.That(warnings[0], Does.StartWith($"ObjectStore: key '{key}' is not supported:")); - } - - [Test] - public void SavingToAnExistingUnsupportedKeyGoesToTheSanitizedNameWithAWarningEveryTime() - { - // an uploaded object under an unsupported name, which the algorithm loads, retrains and saves back - var filePath = Path.Combine(TestStorageRoot, "model.tar.gz"); - File.WriteAllText(filePath, "old"); - var warnings = new List(); - using (var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add)) - { - store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); - Assert.AreEqual("old", store.Read("model.tar.gz")); - - Assert.IsTrue(store.Save("model.tar.gz", "new")); - - Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); - Assert.That(warnings[0], Does.StartWith("ObjectStore: key 'model.tar.gz' is not supported: it has 2 '.' but at most one '.extension' is allowed.")); - Assert.AreEqual("ObjectStore: key 'model.tar.gz' is not supported: the object stored under that name is left as is," + - " this save goes to 'model_tar.gz'", warnings[1]); - // read after write within the run follows the save - Assert.AreEqual("new", store.Read("model.tar.gz")); - Assert.AreEqual("new", store.Read("model_tar.gz")); - Assert.AreEqual("old", File.ReadAllText(filePath)); - Assert.IsTrue(store.Keys.Contains("model.tar.gz") && store.Keys.Contains("model_tar.gz"), string.Join(", ", store.Keys)); - - // every such save warns, not only the first - Assert.IsTrue(store.Save("model.tar.gz", "newer")); - Assert.AreEqual(3, warnings.Count, string.Join("\n", warnings)); - Assert.AreEqual(warnings[1], warnings[2]); - Assert.AreEqual("newer", store.Read("model.tar.gz")); - } - - // a fresh store, the next run: the object under the name as given comes first again, and the save finds - // the previous run's object under the sanitized name, which nothing ties to this key any more - warnings.Clear(); - using (var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add)) - { - store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); - Assert.AreEqual("old", store.Read("model.tar.gz")); - Assert.IsEmpty(warnings); - - var exception = Assert.Throws(() => store.Save("model.tar.gz", "third")); - Assert.AreEqual("ObjectStore: key 'model.tar.gz' is not supported and its sanitized name 'model_tar.gz' already holds" + - " another object. Delete 'model_tar.gz' or save to it directly to replace it", exception.Message); - Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); - Assert.AreEqual("newer", store.Read("model_tar.gz")); - - // once the sanitized name is free the save goes there, warning about the original every time - Assert.IsTrue(store.Delete("model_tar.gz")); - Assert.IsTrue(store.Save("model.tar.gz", "third")); - Assert.AreEqual(2, warnings.Count, string.Join("\n", warnings)); - Assert.That(warnings[1], Does.Contain("the object stored under that name is left as is, this save goes to 'model_tar.gz'")); - Assert.AreEqual("third", store.Read("model.tar.gz")); - Assert.AreEqual("third", store.Read("model_tar.gz")); - - // within the run the key follows the save: the first delete removes it and the original is back - Assert.IsTrue(store.Delete("model.tar.gz")); - Assert.IsFalse(store.ContainsKey("model_tar.gz")); - Assert.AreEqual("old", File.ReadAllText(filePath)); - Assert.AreEqual("old", store.Read("model.tar.gz")); - Assert.IsTrue(store.Delete("model.tar.gz")); - Assert.IsFalse(store.ContainsKey("model.tar.gz")); - } - Assert.IsFalse(File.Exists(filePath)); - } - - [Test] - public void DeletingASavedUnsupportedKeyRemovesTheSavedObjectThenTheOriginal() - { - var filePath = Path.Combine(TestStorageRoot, "model.tar.gz"); - File.WriteAllText(filePath, "old"); - using var store = new ObjectStore(new TestLocalObjectStore(), _ => { }); - store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); - Assert.IsTrue(store.Save("model.tar.gz", "new")); - Assert.AreEqual("new", store.Read("model.tar.gz")); - - // the first delete removes what this store saved, and the key resolves like a fresh store's from then on - Assert.IsTrue(store.Delete("model.tar.gz")); - Assert.IsFalse(File.Exists(Path.Combine(TestStorageRoot, "model_tar.gz"))); - Assert.IsTrue(store.ContainsKey("model.tar.gz")); - Assert.AreEqual("old", store.Read("model.tar.gz")); - - Assert.IsTrue(store.Delete("model.tar.gz")); - Assert.IsFalse(store.ContainsKey("model.tar.gz")); - Assert.IsFalse(File.Exists(filePath)); - } - - [Test] - public void AFailedSaveToAnUnsupportedKeyDoesNotRedirectItsReads() - { - var filePath = Path.Combine(TestStorageRoot, "model.tar.gz"); - File.WriteAllText(filePath, "old"); - using var store = new ObjectStore(new TestLocalObjectStore(), _ => { }); - // the 3 bytes stored fill the storage: the save is rejected, not thrown, in backtesting - store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1, StorageLimit = 3 }, AlgorithmMode.Backtesting); - - Assert.IsFalse(store.Save("model.tar.gz", "new")); - - Assert.IsTrue(store.ContainsKey("model.tar.gz")); - Assert.AreEqual("old", store.Read("model.tar.gz")); - Assert.IsFalse(store.ContainsKey("model_tar.gz")); - - Assert.IsTrue(store.Delete("model.tar.gz")); - Assert.IsFalse(File.Exists(filePath)); - } - - [Test] - public void CustomDataIsReadFromAnExistingUnsupportedKey() - { - using var store = new ObjectStore(new TestLocalObjectStore()); - store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); - File.WriteAllText(Path.Combine(TestStorageRoot, "BRK.B.csv"), "2013-10-07,167.42\n2013-10-08,165.48\n"); - - // the data feed reads custom data through the algorithm's store - using var reader = new ObjectStoreSubscriptionStreamReader(store, "BRK.B.csv"); - - Assert.IsFalse(reader.EndOfStream); - Assert.AreEqual("2013-10-07,167.42", reader.ReadLine()); - Assert.AreEqual("2013-10-08,165.48", reader.ReadLine()); - store.Delete("BRK.B.csv"); - } - - [Test] - public void SavingAnUnsupportedKeyWhoseSanitizedNameHoldsAnotherObjectThrowsEveryTime() - { - var warnings = new List(); - using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); - store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); - Assert.IsTrue(store.Save("trade_log_ai_cloud.csv", "existing")); - Assert.IsEmpty(warnings); - - // sanitizes to the existing object's key: the general warning, then the exception - var exception = Assert.Throws(() => store.Save("trade_log_ai&cloud.csv", "first")); - Assert.AreEqual("ObjectStore: key 'trade_log_ai&cloud.csv' is not supported and its sanitized name 'trade_log_ai_cloud.csv' " + - "already holds another object. Delete 'trade_log_ai_cloud.csv' or save to it directly to replace it", exception.Message); - Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); - Assert.That(warnings[0], Does.StartWith("ObjectStore: key 'trade_log_ai&cloud.csv' is not supported: unsupported character '&'.")); - Assert.AreEqual("existing", store.Read("trade_log_ai_cloud.csv")); - - // every such save throws, from this key or another one sanitizing to the same name - Assert.Throws(() => store.Save("trade_log_ai&cloud.csv", "first")); - exception = Assert.Throws(() => store.Save("trade_log_ai:cloud.csv", "second")); - Assert.AreEqual("ObjectStore: key 'trade_log_ai:cloud.csv' is not supported and its sanitized name 'trade_log_ai_cloud.csv' " + - "already holds another object. Delete 'trade_log_ai_cloud.csv' or save to it directly to replace it", exception.Message); - Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); - Assert.AreEqual("existing", store.Read("trade_log_ai_cloud.csv")); - - // once the name is free the key is saved there and updated silently - Assert.IsTrue(store.Delete("trade_log_ai_cloud.csv")); - Assert.IsTrue(store.Save("trade_log_ai&cloud.csv", "first")); - Assert.IsTrue(store.Save("trade_log_ai&cloud.csv", "second")); - Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); - Assert.AreEqual("second", store.Read("trade_log_ai_cloud.csv")); - Assert.AreEqual(1, store.Keys.Count(key => key.StartsWith("trade_log_ai"))); - - Assert.IsTrue(store.Delete("trade_log_ai&cloud.csv")); - Assert.IsFalse(store.ContainsKey("trade_log_ai_cloud.csv")); - } - - [Test] - public void ANewRunThrowsOnSavingAnUnsupportedKeyAPreviousRunSanitized() - { - // nothing records that a previous run wrote the sanitized object, so no later run replaces it through the unsupported key - using (var firstRun = new ObjectStore(new TestLocalObjectStore(), _ => { })) - { - firstRun.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); - Assert.IsTrue(firstRun.Save("trade log: AI & Cloud.csv", "run1")); - } - - using (var secondRun = new ObjectStore(new TestLocalObjectStore(), _ => { })) - { - secondRun.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); - var exception = Assert.Throws(() => secondRun.Save("trade log: AI & Cloud.csv", "run2")); - Assert.AreEqual("ObjectStore: key 'trade log: AI & Cloud.csv' is not supported and its sanitized name 'trade log_ AI _ Cloud.csv'" + - " already holds another object. Delete 'trade log_ AI _ Cloud.csv' or save to it directly to replace it", exception.Message); - Assert.AreEqual("run1", secondRun.Read("trade log: AI & Cloud.csv")); - - // the exact sanitized key is a supported key: the user replaces the object through it - Assert.IsTrue(secondRun.Save("trade log_ AI _ Cloud.csv", "run2")); - Assert.AreEqual("run2", secondRun.Read("trade log: AI & Cloud.csv")); - Assert.Throws(() => secondRun.Save("trade log: AI & Cloud.csv", "run2 again")); - Assert.IsTrue(secondRun.Delete("trade log_ AI _ Cloud.csv")); - } - } - - [Test] - public void ASaveToTheSanitizedNameFromAnotherKeyEndsTheSilentUpdatesOfAnUnsupportedKey() - { - var warnings = new List(); - using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); - store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); - - Assert.IsTrue(store.Save("p&q.csv", "A1")); - Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); - - // another unsupported key sanitizing to the same name throws, and the first key keeps updating silently - var exception = Assert.Throws(() => store.Save("p:q.csv", "B1")); - Assert.That(exception.Message, Does.StartWith("ObjectStore: key 'p:q.csv' is not supported and its sanitized name 'p_q.csv' already holds another object")); - Assert.AreEqual("A1", store.Read("p_q.csv")); - Assert.IsTrue(store.Save("p&q.csv", "A2")); - Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); - Assert.AreEqual("A2", store.Read("p_q.csv")); - - // a save through the valid name takes the object over: the unsupported key now throws - Assert.IsTrue(store.Save("p_q.csv", "C1")); - exception = Assert.Throws(() => store.Save("p&q.csv", "A3")); - Assert.That(exception.Message, Does.StartWith("ObjectStore: key 'p&q.csv' is not supported and its sanitized name 'p_q.csv' already holds another object")); - Assert.AreEqual("C1", store.Read("p_q.csv")); - - Assert.IsTrue(store.Delete("p_q.csv")); - Assert.IsTrue(store.Save("p&q.csv", "A3")); - Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); - Assert.AreEqual("A3", store.Read("p&q.csv")); - Assert.IsTrue(store.Delete("p&q.csv")); - Assert.IsFalse(store.ContainsKey("p_q.csv")); - } - - [Test] - public void QuantBookWarnsThroughTheEngineLog() - { - var traces = new List(); - Log.LogHandler = new FunctionalLogHandler(null, traces.Add, null); - var quantBook = new QuantBook(); - Assert.AreEqual(typeof(ObjectStore), quantBook.ObjectStore.GetType()); - - Assert.IsTrue(quantBook.ObjectStore.Save("trade log: AI & Cloud.csv", "abc")); - - Assert.AreEqual(1, traces.Count(trace => trace.Contains("ObjectStore: key 'trade log: AI & Cloud.csv' is not supported: unsupported characters ':', '&'.")), - string.Join("\n", traces)); - quantBook.ObjectStore.Delete("trade log: AI & Cloud.csv"); - } - [Test] public void QuantBookUsesThePythonObjectStoreInPythonNotebooks() { @@ -952,38 +601,6 @@ public void QuantBookUsesThePythonObjectStoreInPythonNotebooks() } } - [Test] - public void LocalObjectStoreRejectsUnsupportedKeysStatingTheIssue() - { - // the implementation is the last line of defense, only the wrapper sanitizes - using var store = new TestLocalObjectStore(); - store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }); - - var error = Assert.Throws(() => store.SaveBytes("trade_log_ai_hardware_&_cloud.csv", new byte[] { 1 })); - - Assert.AreEqual("ObjectStore: path is not supported: 'trade_log_ai_hardware_&_cloud.csv': unsupported character '&'. " + - "Keys may only contain english letters, numbers, spaces and the characters '/', '\\', '_', '#', '-', '$' and '=', " + - "plus at most one '.' followed by a letters-and-numbers-only extension, e.g. 'folder/trade_log-2024.csv'", error.Message); - } - - [Test] - public void AlgorithmWrapsTheStoreAndWarnsThroughDebug() - { - using var localStore = new TestLocalObjectStore(); - localStore.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }); - var algorithm = new QCAlgorithm(); - - algorithm.SetObjectStore(localStore); - - Assert.AreEqual(typeof(ObjectStore), algorithm.ObjectStore.GetType()); - Assert.IsTrue(algorithm.ObjectStore.Save("trade_log_ai_hardware_&_cloud.csv", "abc")); - Assert.IsTrue(algorithm.DebugMessages.TryDequeue(out var warning)); - Assert.That(warning, Does.Contain("ObjectStore: key 'trade_log_ai_hardware_&_cloud.csv' is not supported: unsupported character '&'.")); - Assert.That(warning, Does.Contain("Using 'trade_log_ai_hardware___cloud.csv' instead")); - - algorithm.ObjectStore.Delete("trade_log_ai_hardware_&_cloud.csv"); - } - [Test] public void AlgorithmKeepsAnAlreadyWrappedStore() { @@ -1007,13 +624,20 @@ public void SaveTextAndReadTextAreAliasesOfSaveAndRead() [TestCase("my_key", "./LocalObjectStoreTests/my_key")] [TestCase("test/123", "./LocalObjectStoreTests/test/123")] - [TestCase("**abc**", "./LocalObjectStoreTests/__abc__")] - [TestCase("", "./LocalObjectStoreTests/_random_")] - [TestCase("|", "./LocalObjectStoreTests/_")] + [TestCase("**abc**", null)] + [TestCase("", null)] + [TestCase("|", null)] public void GetFilePathReturnsFileName(string key, string expectedRelativePath) { - var expectedPath = Path.GetFullPath(expectedRelativePath).Replace("\\", "/"); - Assert.AreEqual(expectedPath, _store.GetFilePath(key).Replace("\\", "/")); + if (expectedRelativePath == null) + { + Assert.Throws(() => _store.GetFilePath(key)); + } + else + { + var expectedPath = Path.GetFullPath(expectedRelativePath).Replace("\\", "/"); + Assert.AreEqual(expectedPath, _store.GetFilePath(key).Replace("\\", "/")); + } } [Test] @@ -1789,19 +1413,6 @@ def test(object_store): "); } - [Test] - public void PythonHelpersSanitizeUnsupportedKeys() - { - RunPythonTest(@" -def test(object_store): - object_store.save_json('ergonomics/trade log: AI & Cloud.json', {'a': 1}) - assert 'ergonomics/trade log_ AI _ Cloud.json' in list(object_store.keys), list(object_store.keys) - assert object_store.read_json('ergonomics/trade log: AI & Cloud.json') == {'a': 1} - object_store.delete('ergonomics/trade log: AI & Cloud.json') - assert not object_store.contains_key('ergonomics/trade log_ AI _ Cloud.json') -"); - } - /// /// Runs the 'test(object_store)' function of the given Python source against the test store /// diff --git a/Tests/Python/AlgorithmPythonWrapperTests.cs b/Tests/Python/AlgorithmPythonWrapperTests.cs index 902fcb37883c..2dcab23c97a5 100644 --- a/Tests/Python/AlgorithmPythonWrapperTests.cs +++ b/Tests/Python/AlgorithmPythonWrapperTests.cs @@ -153,26 +153,23 @@ public void CallOnEndOfDayExceptionWithParameter() } [Test] - public void PythonAlgorithmGetsAPythonObjectStoreWarningThroughDebug() + public void PythonAlgorithmGetsAPythonObjectStore() { using (Py.GIL()) { - var algorithm = GetAlgorithm("def on_end_of_algorithm(self): self.object_store.save_json('trade log: AI & Cloud.json', {'a': 1})"); + var algorithm = GetAlgorithm("def on_end_of_algorithm(self): self.object_store.save_json('trade_log.json', {'a': 1})"); using var localStore = new LocalObjectStore(); localStore.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); algorithm.SetObjectStore(localStore); Assert.IsInstanceOf(algorithm.ObjectStore); - // the Python helpers are reachable through self.object_store and the key warning lands in the algorithm's log + // the Python helpers are reachable through self.object_store algorithm.OnEndOfAlgorithm(); Assert.Null(algorithm.RunTimeError); - Assert.IsTrue(localStore.Keys.Contains("trade log_ AI _ Cloud.json"), string.Join(", ", localStore.Keys)); - Assert.IsTrue(algorithm.BaseAlgorithm.DebugMessages.TryDequeue(out var warning)); - Assert.That(warning, Does.Contain("ObjectStore: key 'trade log: AI & Cloud.json' is not supported: unsupported characters ':', '&'.")); - Assert.That(warning, Does.Contain("Using 'trade log_ AI _ Cloud.json' instead")); + Assert.AreEqual("{\"a\": 1}", algorithm.ObjectStore.Read("trade_log.json")); - localStore.Delete("trade log_ AI _ Cloud.json"); + localStore.Delete("trade_log.json"); } } From 9306dbcf0f9ba4df6dca038fc98cf05c07e4a3af Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 4 Nov 2026 14:25:25 -0400 Subject: [PATCH 14/14] Rename ergonomics to helpers in the regression algorithm and tests --- ... ObjectStoreHelpersRegressionAlgorithm.cs} | 16 ++-- ... ObjectStoreHelpersRegressionAlgorithm.py} | 28 +++--- Tests/Common/Storage/LocalObjectStoreTests.cs | 96 +++++++++---------- 3 files changed, 70 insertions(+), 70 deletions(-) rename Algorithm.CSharp/{ObjectStoreErgonomicsRegressionAlgorithm.cs => ObjectStoreHelpersRegressionAlgorithm.cs} (88%) rename Algorithm.Python/{ObjectStoreErgonomicsRegressionAlgorithm.py => ObjectStoreHelpersRegressionAlgorithm.py} (79%) diff --git a/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs b/Algorithm.CSharp/ObjectStoreHelpersRegressionAlgorithm.cs similarity index 88% rename from Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs rename to Algorithm.CSharp/ObjectStoreHelpersRegressionAlgorithm.cs index ba40d977d328..b36d49937d00 100644 --- a/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs +++ b/Algorithm.CSharp/ObjectStoreHelpersRegressionAlgorithm.cs @@ -20,12 +20,12 @@ namespace QuantConnect.Algorithm.CSharp { /// - /// Regression algorithm asserting the ObjectStore ergonomics: and + /// Regression algorithm asserting the ObjectStore helpers: and /// are aliases of Save and Read, and reading a missing key lists the /// available keys in the error. The Python version of this algorithm additionally covers the tolerant /// save_json/read_json and save_dataframe/read_dataframe helpers /// - public class ObjectStoreErgonomicsRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + public class ObjectStoreHelpersRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition { /// /// Initialize the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. @@ -46,9 +46,9 @@ public override void Initialize() public override void OnEndOfAlgorithm() { // SaveText and ReadText are aliases of Save and Read - if (!ObjectStore.SaveText("ergonomics_report.txt", "The strategy went up") || - ObjectStore.Read("ergonomics_report.txt") != "The strategy went up" || - ObjectStore.ReadText("ergonomics_report.txt") != "The strategy went up") + if (!ObjectStore.SaveText("helpers_report.txt", "The strategy went up") || + ObjectStore.Read("helpers_report.txt") != "The strategy went up" || + ObjectStore.ReadText("helpers_report.txt") != "The strategy went up") { throw new RegressionTestException("Expected the SaveText/ReadText round trip to succeed"); } @@ -57,18 +57,18 @@ public override void OnEndOfAlgorithm() var errorMessage = string.Empty; try { - ObjectStore.ReadBytes("ergonomics_missing.json"); + ObjectStore.ReadBytes("helpers_missing.json"); } catch (KeyNotFoundException exception) { errorMessage = exception.Message; } - if (!errorMessage.Contains("Keys: [") || !errorMessage.Contains("'ergonomics_report.txt'")) + if (!errorMessage.Contains("Keys: [") || !errorMessage.Contains("'helpers_report.txt'")) { throw new RegressionTestException($"Expected the available keys in the missing key error, got: '{errorMessage}'"); } - ObjectStore.Delete("ergonomics_report.txt"); + ObjectStore.Delete("helpers_report.txt"); } /// diff --git a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py b/Algorithm.Python/ObjectStoreHelpersRegressionAlgorithm.py similarity index 79% rename from Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py rename to Algorithm.Python/ObjectStoreHelpersRegressionAlgorithm.py index 5a5b73147dc5..c215cc45052b 100644 --- a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py +++ b/Algorithm.Python/ObjectStoreHelpersRegressionAlgorithm.py @@ -16,11 +16,11 @@ from decimal import Decimal ### -### Regression algorithm asserting the ObjectStore ergonomics: save_text and read_text are aliases of save and read, +### Regression algorithm asserting the ObjectStore helpers: save_text and read_text are aliases of save and read, ### reading a missing key lists the available keys in the error, and the tolerant save_json/read_json and ### save_dataframe/read_dataframe helpers handle datetime/date/Decimal/Symbol and data frames out of the box. ### -class ObjectStoreErgonomicsRegressionAlgorithm(QCAlgorithm): +class ObjectStoreHelpersRegressionAlgorithm(QCAlgorithm): def initialize(self): self.set_start_date(2013, 10, 7) @@ -42,9 +42,9 @@ def on_end_of_algorithm(self): "returns": np.array([0.01, np.nan]), "bar_times": pd.DatetimeIndex([datetime(2013, 10, 10), datetime(2013, 10, 11)]).values } - if not self.object_store.save_json("ergonomics/trade_log.json", trade_log): + if not self.object_store.save_json("helpers/trade_log.json", trade_log): raise AssertionError("Expected save_json to succeed") - data = self.object_store.read_json("ergonomics/trade_log.json") + data = self.object_store.read_json("helpers/trade_log.json") expected = { "time": "2013-10-11T16:00:00", "date": "2013-10-11", @@ -64,7 +64,7 @@ def on_end_of_algorithm(self): # save_json refuses what it cannot serialize faithfully instead of storing its string error = None try: - self.object_store.save_json("ergonomics/rejected.json", {"history": pd.Series([1.0, 2.0])}) + self.object_store.save_json("helpers/rejected.json", {"history": pd.Series([1.0, 2.0])}) except TypeError as e: error = str(e) if error is None or "save_dataframe" not in error: @@ -73,7 +73,7 @@ def on_end_of_algorithm(self): # read_json throws on a missing key like every other read method error = None try: - self.object_store.read_json("ergonomics/missing.json") + self.object_store.read_json("helpers/missing.json") except KeyNotFoundException as e: error = str(e) if error is None: @@ -81,29 +81,29 @@ def on_end_of_algorithm(self): # save_dataframe stores a pandas DataFrame as CSV and read_dataframe restores it, time index included frame = pd.DataFrame({"close": [167.42, 168.0]}, index=pd.to_datetime(["2013-10-10", "2013-10-11"])) - if not self.object_store.save_dataframe("ergonomics/history.csv", frame): + if not self.object_store.save_dataframe("helpers/history.csv", frame): raise AssertionError("Expected save_dataframe to succeed") - csv = self.object_store.read("ergonomics/history.csv") + csv = self.object_store.read("helpers/history.csv") if "close" not in csv or "2013-10-10" not in csv: raise AssertionError(f"Unexpected save_dataframe content: {csv}") - restored = self.object_store.read_dataframe("ergonomics/history.csv") + restored = self.object_store.read_dataframe("helpers/history.csv") if not restored.equals(frame): raise AssertionError(f"Unexpected read_dataframe round trip result: {restored}") # save_text/read_text are aliases of save/read - if not self.object_store.save_text("ergonomics/report.txt", "The strategy went up"): + if not self.object_store.save_text("helpers/report.txt", "The strategy went up"): raise AssertionError("Expected save_text to succeed") - if self.object_store.read_text("ergonomics/report.txt") != "The strategy went up": + if self.object_store.read_text("helpers/report.txt") != "The strategy went up": raise AssertionError("Expected the save_text/read_text round trip to succeed") # reading a missing key lists the available keys in the error error = None try: - self.object_store.read_bytes("ergonomics/missing.json") + self.object_store.read_bytes("helpers/missing.json") except BaseException as e: error = str(e) - if error is None or "Keys: [" not in error or "'ergonomics/report.txt'" not in error: + if error is None or "Keys: [" not in error or "'helpers/report.txt'" not in error: raise AssertionError(f"Expected the available keys in the missing key error, got: '{error}'") - for key in ["ergonomics/trade_log.json", "ergonomics/history.csv", "ergonomics/report.txt"]: + for key in ["helpers/trade_log.json", "helpers/history.csv", "helpers/report.txt"]: self.object_store.delete(key) diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index b1a42909d1ae..4f6b6e42892a 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -1105,14 +1105,14 @@ def test(object_store): symbol = Symbol.create('SPY', SecurityType.EQUITY, Market.USA) # datetime/date/Decimal/Symbol values and non-string dictionary keys, which json.dumps rejects - object_store.save_json('ergonomics/log.json', { + object_store.save_json('helpers/log.json', { 'time': datetime(2024, 1, 2, 3, 4, 5), 'date': datetime(2024, 1, 1).date(), 'qty': Decimal('1.5'), 'symbol': symbol, 'fills': [{symbol: 10}] }) - data = object_store.read_json('ergonomics/log.json') + data = object_store.read_json('helpers/log.json') # a Symbol value is written like the C# SaveJson does, a Symbol key as its string like a .NET dictionary key assert data == { 'time': '2024-01-02T03:04:05', @@ -1125,7 +1125,7 @@ def test(object_store): assert SymbolCache.get_symbol(data['symbol']['id']) == symbol assert SymbolCache.get_symbol(next(iter(data['fills'][0]))) == symbol - object_store.delete('ergonomics/log.json') + object_store.delete('helpers/log.json') "); } @@ -1142,15 +1142,15 @@ def test(object_store): weights['a'] = 1.5 point = IndicatorDataPoint(datetime(2020, 1, 1), 5) - object_store.save_json('ergonomics/dotnet.json', {'numbers': numbers, 'weights': weights, 'point': point}) - data = object_store.read_json('ergonomics/dotnet.json') + object_store.save_json('helpers/dotnet.json', {'numbers': numbers, 'weights': weights, 'point': point}) + data = object_store.read_json('helpers/dotnet.json') assert data == { 'numbers': [1, 2], 'weights': {'a': 1.5}, 'point': json.loads(newtonsoft.JsonConvert.serialize_object(point)) }, data - object_store.delete('ergonomics/dotnet.json') + object_store.delete('helpers/dotnet.json') "); } @@ -1161,16 +1161,16 @@ public void PythonSaveJsonConvertsNumpyArraysAndScalars() import numpy as np def test(object_store): - object_store.save_json('ergonomics/numpy.json', { + object_store.save_json('helpers/numpy.json', { 'weights': np.array([1.5, 2.5]), 'grid': np.array([[1, 2], [3, 4]]), 'count': np.int64(3), 'ratio': np.float32(0.5) }) - data = object_store.read_json('ergonomics/numpy.json') + data = object_store.read_json('helpers/numpy.json') assert data == {'weights': [1.5, 2.5], 'grid': [[1, 2], [3, 4]], 'count': 3, 'ratio': 0.5}, data - object_store.delete('ergonomics/numpy.json') + object_store.delete('helpers/numpy.json') "); } @@ -1181,19 +1181,19 @@ public void PythonSaveJsonWritesNanAndInfinityAsNull() import numpy as np def test(object_store): - object_store.save_json('ergonomics/nan.json', { + object_store.save_json('helpers/nan.json', { 'pnl': float('nan'), 'ratio': float('inf'), 'drawdown': -float('inf'), 'series': np.array([1.0, np.nan]), 'scalar': np.float64('nan') }) - text = object_store.read('ergonomics/nan.json') + text = object_store.read('helpers/nan.json') assert 'NaN' not in text and 'Infinity' not in text, text - data = object_store.read_json('ergonomics/nan.json') + data = object_store.read_json('helpers/nan.json') assert data == {'pnl': None, 'ratio': None, 'drawdown': None, 'series': [1.0, None], 'scalar': None}, data - object_store.delete('ergonomics/nan.json') + object_store.delete('helpers/nan.json') "); } @@ -1204,7 +1204,7 @@ public void PythonSaveJsonWritesNumpyDatetimesAsIso8601AndNatAsNull() import numpy as np def test(object_store): - object_store.save_json('ergonomics/datetime64.json', { + object_store.save_json('helpers/datetime64.json', { 'index': pd.DatetimeIndex([pd.Timestamp('2020-01-01'), pd.Timestamp('2020-01-02 03:04:05.000000006')]).values, 'scalar': np.datetime64('2020-01-01T00:00:00.000000000'), 'day': np.datetime64('2020-01-01'), @@ -1214,7 +1214,7 @@ def test(object_store): 'with_nat': pd.to_datetime(['2020-01-01', None]).values, 'grid': np.array([['2020-01-01', '2020-01-02']], dtype='datetime64[D]') }) - data = object_store.read_json('ergonomics/datetime64.json') + data = object_store.read_json('helpers/datetime64.json') assert data == { 'index': ['2020-01-01T00:00:00', '2020-01-02T03:04:05.000000006'], 'scalar': '2020-01-01T00:00:00', @@ -1226,7 +1226,7 @@ def test(object_store): 'grid': [['2020-01-01T00:00:00', '2020-01-02T00:00:00']] }, data - object_store.delete('ergonomics/datetime64.json') + object_store.delete('helpers/datetime64.json') "); } @@ -1238,14 +1238,14 @@ import numpy as np from datetime import timedelta def test(object_store): - object_store.save_json('ergonomics/timedelta.json', { + object_store.save_json('helpers/timedelta.json', { 'holding': timedelta(days=1, hours=2), 'latency': np.timedelta64(1500, 'ms'), 'spans': (pd.to_datetime(['2020-01-02', '2020-01-03']) - pd.Timestamp('2020-01-01')).values, 'pandas': pd.Timedelta(seconds=1.5), 'nat': np.timedelta64('NaT') }) - data = object_store.read_json('ergonomics/timedelta.json') + data = object_store.read_json('helpers/timedelta.json') assert data == { 'holding': 'P1DT2H0M0S', 'latency': 'P0DT0H0M1.5S', @@ -1255,7 +1255,7 @@ def test(object_store): }, data assert pd.Timedelta(data['holding']) == timedelta(days=1, hours=2) - object_store.delete('ergonomics/timedelta.json') + object_store.delete('helpers/timedelta.json') "); } @@ -1266,11 +1266,11 @@ public void PythonSaveJsonRejectsNanDictionaryKeysWithTypeError() def test(object_store): error = None try: - object_store.save_json('ergonomics/nan_key.json', {float('nan'): 1}) + object_store.save_json('helpers/nan_key.json', {float('nan'): 1}) except TypeError as e: error = str(e) assert error is not None and 'nan' in error and 'key' in error, error - assert not object_store.contains_key('ergonomics/nan_key.json') + assert not object_store.contains_key('helpers/nan_key.json') "); } @@ -1282,11 +1282,11 @@ def test(object_store): for value in [pd.DataFrame({'a': [1, 2]}), pd.Series([1, 2])]: error = None try: - object_store.save_json('ergonomics/pandas.json', {'data': value}) + object_store.save_json('helpers/pandas.json', {'data': value}) except TypeError as e: error = str(e) assert error is not None and 'save_dataframe' in error and type(value).__name__ in error, error - assert not object_store.contains_key('ergonomics/pandas.json') + assert not object_store.contains_key('helpers/pandas.json') "); } @@ -1301,11 +1301,11 @@ def test(object_store): for value in [TradeRecord(), object(), (x for x in [1])]: error = None try: - object_store.save_json('ergonomics/unsupported.json', {'data': value}) + object_store.save_json('helpers/unsupported.json', {'data': value}) except TypeError as e: error = str(e) assert error is not None and type(value).__name__ in error, error - assert not object_store.contains_key('ergonomics/unsupported.json') + assert not object_store.contains_key('helpers/unsupported.json') "); } @@ -1318,23 +1318,23 @@ def test(object_store): try: for value in [None, [1, 'a'], (1, 2)]: expected = list(value) if isinstance(value, tuple) else value - object_store.save_json('ergonomics/container.json', value) - data = object_store.read_json('ergonomics/container.json') + object_store.save_json('helpers/container.json', value) + data = object_store.read_json('helpers/container.json') if data != expected: failures.append(f'{value!r} -> {data!r}') # a top-level str/int/float/bool does not bind to the PyObject overload: the error names the mismatch for value in ['text', 42, 1.5, True]: try: - object_store.save_json('ergonomics/scalar.json', value) + object_store.save_json('helpers/scalar.json', value) failures.append(f'{value!r} was accepted') except TypeError as e: if 'No method matches' not in str(e): failures.append(f'{value!r} -> {e}') - if object_store.contains_key('ergonomics/scalar.json'): + if object_store.contains_key('helpers/scalar.json'): failures.append('a rejected scalar was saved') finally: - for key in ['ergonomics/container.json', 'ergonomics/scalar.json']: + for key in ['helpers/container.json', 'helpers/scalar.json']: if object_store.contains_key(key): object_store.delete(key) assert not failures, failures @@ -1346,7 +1346,7 @@ public void PythonReadJsonThrowsOnMissingKey() { RunPythonTest(@" def test(object_store): - assert_raises_key_not_found(lambda: object_store.read_json('ergonomics/missing.json')) + assert_raises_key_not_found(lambda: object_store.read_json('helpers/missing.json')) "); } @@ -1355,11 +1355,11 @@ public void PythonSaveTextAndReadTextAreAliasesOfSaveAndRead() { RunPythonTest(@" def test(object_store): - object_store.save_text('ergonomics/report.txt', 'hello') - assert object_store.read('ergonomics/report.txt') == 'hello' - assert object_store.read_text('ergonomics/report.txt') == 'hello' + object_store.save_text('helpers/report.txt', 'hello') + assert object_store.read('helpers/report.txt') == 'hello' + assert object_store.read_text('helpers/report.txt') == 'hello' - object_store.delete('ergonomics/report.txt') + object_store.delete('helpers/report.txt') "); } @@ -1369,11 +1369,11 @@ public void PythonSaveDataframeAndReadDataframeRoundTripAsCsv() RunPythonTest(@" def test(object_store): frame = pd.DataFrame({'a': [1, 2], 'b': [3.5, 4.5]}) - object_store.save_dataframe('ergonomics/frame.csv', frame) - assert 'a,b' in object_store.read('ergonomics/frame.csv') - pd.testing.assert_frame_equal(object_store.read_dataframe('ergonomics/frame.csv'), frame) + object_store.save_dataframe('helpers/frame.csv', frame) + assert 'a,b' in object_store.read('helpers/frame.csv') + pd.testing.assert_frame_equal(object_store.read_dataframe('helpers/frame.csv'), frame) - object_store.delete('ergonomics/frame.csv') + object_store.delete('helpers/frame.csv') "); } @@ -1383,11 +1383,11 @@ public void PythonSaveDataframeAndReadDataframeRoundTripAsJson() RunPythonTest(@" def test(object_store): frame = pd.DataFrame({'a': [1, 2], 'b': [3.5, 4.5]}) - object_store.save_dataframe('ergonomics/frame.json', frame) - assert object_store.read('ergonomics/frame.json') == frame.to_json() - pd.testing.assert_frame_equal(object_store.read_dataframe('ergonomics/frame.json'), frame) + object_store.save_dataframe('helpers/frame.json', frame) + assert object_store.read('helpers/frame.json') == frame.to_json() + pd.testing.assert_frame_equal(object_store.read_dataframe('helpers/frame.json'), frame) - object_store.delete('ergonomics/frame.json') + object_store.delete('helpers/frame.json') "); } @@ -1397,10 +1397,10 @@ public void PythonReadDataframeRestoresATimeIndex() RunPythonTest(@" def test(object_store): frame = pd.DataFrame({'close': [167.42, 168.0]}, index=pd.to_datetime(['2013-10-10', '2013-10-11'])) - object_store.save_dataframe('ergonomics/history.csv', frame) - pd.testing.assert_frame_equal(object_store.read_dataframe('ergonomics/history.csv'), frame) + object_store.save_dataframe('helpers/history.csv', frame) + pd.testing.assert_frame_equal(object_store.read_dataframe('helpers/history.csv'), frame) - object_store.delete('ergonomics/history.csv') + object_store.delete('helpers/history.csv') "); } @@ -1409,7 +1409,7 @@ public void PythonReadDataframeThrowsOnMissingKey() { RunPythonTest(@" def test(object_store): - assert_raises_key_not_found(lambda: object_store.read_dataframe('ergonomics/missing.csv')) + assert_raises_key_not_found(lambda: object_store.read_dataframe('helpers/missing.csv')) "); } @@ -1420,7 +1420,7 @@ private void RunPythonTest(string source) { using (Py.GIL()) { - var testModule = PyModule.FromString("TestObjectStoreErgonomics", @" + var testModule = PyModule.FromString("TestObjectStoreHelpers", @" from AlgorithmImports import * from System.Collections.Generic import KeyNotFoundException from datetime import datetime