diff --git a/Algorithm.CSharp/ObjectStoreHelpersRegressionAlgorithm.cs b/Algorithm.CSharp/ObjectStoreHelpersRegressionAlgorithm.cs new file mode 100644 index 000000000000..b36d49937d00 --- /dev/null +++ b/Algorithm.CSharp/ObjectStoreHelpersRegressionAlgorithm.cs @@ -0,0 +1,134 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using QuantConnect.Interfaces; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Regression algorithm asserting the ObjectStore helpers: and + /// are aliases of Save and Read, and reading a missing key lists the + /// available keys in the error. The Python version of this algorithm additionally covers the tolerant + /// save_json/read_json and save_dataframe/read_dataframe helpers + /// + public class ObjectStoreHelpersRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + /// + /// Initialize the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. + /// + public override void Initialize() + { + SetStartDate(2013, 10, 7); + SetEndDate(2013, 10, 11); + + SetBenchmark(x => 0); + + AddEquity("SPY", Resolution.Daily); + } + + /// + /// End of algorithm run event handler. This method is called at the end of a backtest or live trading operation. + /// + public override void OnEndOfAlgorithm() + { + // SaveText and ReadText are aliases of Save and Read + if (!ObjectStore.SaveText("helpers_report.txt", "The strategy went up") || + ObjectStore.Read("helpers_report.txt") != "The strategy went up" || + ObjectStore.ReadText("helpers_report.txt") != "The strategy went up") + { + throw new RegressionTestException("Expected the SaveText/ReadText round trip to succeed"); + } + + // reading a missing key lists the available keys in the error + var errorMessage = string.Empty; + try + { + ObjectStore.ReadBytes("helpers_missing.json"); + } + catch (KeyNotFoundException exception) + { + errorMessage = exception.Message; + } + if (!errorMessage.Contains("Keys: [") || !errorMessage.Contains("'helpers_report.txt'")) + { + throw new RegressionTestException($"Expected the available keys in the missing key error, got: '{errorMessage}'"); + } + + ObjectStore.Delete("helpers_report.txt"); + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public List Languages { get; } = new() { Language.CSharp, Language.Python }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public long DataPoints => 6; + + /// + /// Data Points count of the algorithm history + /// + public int AlgorithmHistoryDataPoints => 0; + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "0"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "0%"}, + {"Drawdown", "0%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "100000"}, + {"Net Profit", "0%"}, + {"Sharpe Ratio", "0"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "0%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "0"}, + {"Beta", "0"}, + {"Annual Standard Deviation", "0"}, + {"Annual Variance", "0"}, + {"Information Ratio", "0"}, + {"Tracking Error", "0"}, + {"Treynor Ratio", "0"}, + {"Total Fees", "$0.00"}, + {"Estimated Strategy Capacity", "$0"}, + {"Lowest Capacity Asset", ""}, + {"Portfolio Turnover", "0%"}, + {"Drawdown Recovery", "0"}, + {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"} + }; + } +} diff --git a/Algorithm.Python/ObjectStoreHelpersRegressionAlgorithm.py b/Algorithm.Python/ObjectStoreHelpersRegressionAlgorithm.py new file mode 100644 index 000000000000..c215cc45052b --- /dev/null +++ b/Algorithm.Python/ObjectStoreHelpersRegressionAlgorithm.py @@ -0,0 +1,109 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from AlgorithmImports import * +from System.Collections.Generic import KeyNotFoundException, List +from decimal import Decimal + +### +### Regression algorithm asserting the ObjectStore helpers: save_text and read_text are aliases of save and read, +### reading a missing key lists the available keys in the error, and the tolerant save_json/read_json and +### save_dataframe/read_dataframe helpers handle datetime/date/Decimal/Symbol and data frames out of the box. +### +class ObjectStoreHelpersRegressionAlgorithm(QCAlgorithm): + + def initialize(self): + self.set_start_date(2013, 10, 7) + self.set_end_date(2013, 10, 11) + + self.set_benchmark(lambda x: 0) + + self.spy = self.add_equity("SPY", Resolution.DAILY).symbol + + def on_end_of_algorithm(self): + # save_json tolerates datetime/date/Decimal/numpy/NaN values, .NET objects and non-string dictionary keys + trade_log = { + "time": datetime(2013, 10, 11, 16, 0, 0), + "date": date(2013, 10, 11), + "qty": Decimal("1.5"), + "symbol": self.spy, + "fills": [{"price": Decimal("167.42"), self.spy: 10}], + "fill_quantities": List[int]([10, -10]), + "returns": np.array([0.01, np.nan]), + "bar_times": pd.DatetimeIndex([datetime(2013, 10, 10), datetime(2013, 10, 11)]).values + } + if not self.object_store.save_json("helpers/trade_log.json", trade_log): + raise AssertionError("Expected save_json to succeed") + data = self.object_store.read_json("helpers/trade_log.json") + expected = { + "time": "2013-10-11T16:00:00", + "date": "2013-10-11", + "qty": 1.5, + "symbol": {"value": "SPY", "id": str(self.spy.id), "permtick": "SPY"}, + "fills": [{"price": 167.42, "SPY": 10}], + "fill_quantities": [10, -10], + "returns": [0.01, None], + "bar_times": ["2013-10-10T00:00:00", "2013-10-11T00:00:00"] + } + if data != expected: + raise AssertionError(f"Unexpected read_json round trip result: {data}") + # the symbol id and the ticker key both resolve back to the added security through the symbol cache + if self.symbol(data["symbol"]["id"]) != self.spy or self.symbol("SPY") != self.spy: + raise AssertionError(f"Expected the stored symbol to resolve to {self.spy}, got: {data['symbol']}") + + # save_json refuses what it cannot serialize faithfully instead of storing its string + error = None + try: + self.object_store.save_json("helpers/rejected.json", {"history": pd.Series([1.0, 2.0])}) + except TypeError as e: + error = str(e) + if error is None or "save_dataframe" not in error: + raise AssertionError(f"Expected save_json to reject a pandas Series pointing to save_dataframe, got: '{error}'") + + # read_json throws on a missing key like every other read method + error = None + try: + self.object_store.read_json("helpers/missing.json") + except KeyNotFoundException as e: + error = str(e) + if error is None: + raise AssertionError("Expected read_json of a missing key to throw") + + # save_dataframe stores a pandas DataFrame as CSV and read_dataframe restores it, time index included + frame = pd.DataFrame({"close": [167.42, 168.0]}, index=pd.to_datetime(["2013-10-10", "2013-10-11"])) + if not self.object_store.save_dataframe("helpers/history.csv", frame): + raise AssertionError("Expected save_dataframe to succeed") + csv = self.object_store.read("helpers/history.csv") + if "close" not in csv or "2013-10-10" not in csv: + raise AssertionError(f"Unexpected save_dataframe content: {csv}") + restored = self.object_store.read_dataframe("helpers/history.csv") + if not restored.equals(frame): + raise AssertionError(f"Unexpected read_dataframe round trip result: {restored}") + + # save_text/read_text are aliases of save/read + if not self.object_store.save_text("helpers/report.txt", "The strategy went up"): + raise AssertionError("Expected save_text to succeed") + if self.object_store.read_text("helpers/report.txt") != "The strategy went up": + raise AssertionError("Expected the save_text/read_text round trip to succeed") + + # reading a missing key lists the available keys in the error + error = None + try: + self.object_store.read_bytes("helpers/missing.json") + except BaseException as e: + error = str(e) + if error is None or "Keys: [" not in error or "'helpers/report.txt'" not in error: + raise AssertionError(f"Expected the available keys in the missing key error, got: '{error}'") + + for key in ["helpers/trade_log.json", "helpers/history.csv", "helpers/report.txt"]: + self.object_store.delete(key) diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index 5cefc6375a19..520867b545e7 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -3308,7 +3308,8 @@ public void SetApi(IApi api) [DocumentationAttribute(MachineLearning)] public void SetObjectStore(IObjectStore objectStore) { - ObjectStore = new ObjectStore(objectStore); + // an already wrapped store, like the Python one, is used as is + ObjectStore = objectStore as ObjectStore ?? new ObjectStore(objectStore); } /// diff --git a/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs b/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs index 473aa82cca9b..2fafbc4f675c 100644 --- a/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs +++ b/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs @@ -1231,7 +1231,7 @@ public void SetCurrentSlice(Slice slice) /// Sets the object store /// /// The object store - public void SetObjectStore(IObjectStore objectStore) => _baseAlgorithm.SetObjectStore(objectStore); + public void SetObjectStore(IObjectStore objectStore) => _baseAlgorithm.SetObjectStore(new PythonObjectStore(objectStore)); /// /// Determines if the Symbol is shortable at the brokerage diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index bb4795c00d60..a610c1d00752 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -159,6 +159,17 @@ public string ReadString(string path, Encoding encoding = null) return Read(path, encoding); } + /// + /// Returns the object data for the specified path as a string. Alias of + /// + /// The object path + /// The string encoding used, by default + /// A string containing the data + public string ReadText(string path, Encoding encoding = null) + { + return Read(path, encoding); + } + /// /// Returns the JSON deserialized object data for the specified path /// @@ -239,6 +250,19 @@ public bool SaveString(string path, string text, Encoding encoding = null) return _store.SaveBytes(path, encoding.GetBytes(text)); } + /// + /// Saves the object data in text format for the specified path. + /// Alias of + /// + /// The object path + /// The string object to be saved + /// The string encoding used, by default + /// True if the object was saved successfully + public bool SaveText(string path, string text, Encoding encoding = null) + { + return Save(path, text, encoding); + } + /// /// Saves the object data in JSON format for the specified path /// @@ -299,4 +323,4 @@ public void Dispose() _store.Dispose(); } } -} \ No newline at end of file +} diff --git a/Common/Storage/PythonObjectStore.cs b/Common/Storage/PythonObjectStore.cs new file mode 100644 index 000000000000..a6792db05535 --- /dev/null +++ b/Common/Storage/PythonObjectStore.cs @@ -0,0 +1,220 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Python.Runtime; +using QuantConnect.Interfaces; + +namespace QuantConnect.Storage +{ + /// + /// The of Python algorithms and notebooks: adds the save and read methods that take + /// and return Python objects, such as dictionaries and pandas DataFrames + /// + public class PythonObjectStore : ObjectStore + { + // Python helpers for the PyObject APIs, lazily created under the GIL + private static PyObject _jsonSerializeMethod; + private static PyObject _jsonDeserializeMethod; + private static PyObject _dataFrameSerializeMethod; + private static PyObject _dataFrameDeserializeMethod; + + /// + /// Initializes a new instance of the class + /// + /// The instance to wrap + public PythonObjectStore(IObjectStore store) + : base(store) + { + } + + /// + /// Returns the JSON deserialized object data for the specified path as Python objects + /// + /// The object path + /// The deserialized Python object + public PyObject ReadJson(string path) + { + var json = Read(path); + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var pyJson = json.ToPython(); + return _jsonDeserializeMethod.Invoke(pyJson); + } + } + + /// + /// Returns the object data for the specified path as a pandas DataFrame, read as JSON if the path has + /// a '.json' extension and as CSV otherwise, so a round trip restores the frame. + /// A CSV index in ISO-8601 format is restored as a time index. A saved Series comes back as a + /// one-column DataFrame, call squeeze() on it to get the Series + /// + /// The object path + /// The pandas DataFrame + public PyObject ReadDataframe(string path) + { + var serialized = Read(path); + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var pySerialized = serialized.ToPython(); + using var pyPath = path.ToPython(); + return _dataFrameDeserializeMethod.Invoke(pySerialized, pyPath); + } + } + + /// + /// Saves the given Python dictionary, list, tuple or .NET object in JSON format for the specified path, + /// tolerating nested types the standard json module rejects: datetime/date/time, pandas Timestamp and + /// numpy datetime64 are stored in ISO-8601 format, timedelta and timedelta64 as ISO-8601 durations, + /// Decimal and numpy numbers as numbers, NaN, infinity and NaT as null, .NET objects (Symbol included) + /// as the C# overload writes them (a NaN inside one is Newtonsoft's "NaN" + /// string) and non-string dictionary keys as strings. pandas DataFrames and Series are rejected, use + /// , and so is any other type that would not round trip. A top-level Python + /// str, int, float or bool is not accepted, use for text + /// + /// The object path + /// The Python object to be saved + /// True if the object was saved successfully + /// A TypeError naming the value that cannot be serialized + public bool SaveJson(string path, PyObject obj) + { + string json; + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var result = _jsonSerializeMethod.Invoke(obj); + json = result.As(); + } + return Save(path, json); + } + + /// + /// Saves the given pandas DataFrame (or Series) for the specified path, in JSON format if the + /// path has a '.json' extension and as CSV otherwise + /// + /// The object path + /// The pandas DataFrame or Series to be saved + /// True if the object was saved successfully + public bool SaveDataframe(string path, PyObject dataFrame) + { + string serialized; + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var pyPath = (path ?? string.Empty).ToPython(); + using var result = _dataFrameSerializeMethod.Invoke(dataFrame, pyPath); + serialized = result.As(); + } + return Save(path, serialized); + } + + /// + /// Lazily creates the Python helper methods backing the PyObject APIs. Must be called under the GIL + /// + private static void EnsurePythonHelpers() + { + if (_jsonSerializeMethod == null) + { + var module = PyModule.FromString("object_store_helpers", @"from json import dumps, loads +from datetime import datetime, date, time, timedelta +from decimal import Decimal +from math import isfinite +import numpy +import pandas +import System +import Newtonsoft.Json as newtonsoft + +def _key(key): + # json only accepts str/int/float/bool/None keys; a .NET key is its string, like a .NET dictionary key + if isinstance(key, float) and not isfinite(key): + raise TypeError(f'save_json() cannot use {key} as a dictionary key') + if isinstance(key, (str, int, float)) or key is None: + return key + if isinstance(key, System.Object): + return str(key) + normalized = _normalize(key) + if isinstance(normalized, (str, int, float)) and not isinstance(normalized, bool): + return normalized + raise TypeError(f'save_json() cannot use a {type(key).__name__} as a dictionary key') + +def _normalize(value): + if isinstance(value, dict): + return {_key(k): _normalize(v) for k, v in value.items()} + if isinstance(value, (list, tuple, set, frozenset)): + return [_normalize(v) for v in value] + if isinstance(value, (str, int)) or value is None: + return value + if isinstance(value, float): + # NaN and infinity are not valid JSON + return value if isfinite(value) else None + if isinstance(value, (datetime, date, time)): + return None if value is pandas.NaT else value.isoformat() + if isinstance(value, timedelta): + return pandas.Timedelta(value).isoformat() + if isinstance(value, Decimal): + return _normalize(float(value)) + if isinstance(value, numpy.ndarray): + # tolist() turns nanosecond datetime64/timedelta64 into integers: convert those element-wise + return [_normalize(v) for v in value] if value.dtype.kind in 'mM' else _normalize(value.tolist()) + if isinstance(value, numpy.datetime64): + return None if numpy.isnat(value) else pandas.Timestamp(value).isoformat() + if isinstance(value, numpy.timedelta64): + return None if numpy.isnat(value) else pandas.Timedelta(value).isoformat() + if isinstance(value, numpy.generic): + return _normalize(value.item()) + if isinstance(value, (pandas.DataFrame, pandas.Series)): + raise TypeError(f'save_json() does not accept a pandas {type(value).__name__}, use save_dataframe() instead') + if isinstance(value, System.Object): + # .NET objects serialize like the C# SaveJson does + return loads(newtonsoft.JsonConvert.SerializeObject(value)) + raise TypeError(f'save_json() cannot serialize a {type(value).__name__}: convert it to a dict, list or str first') + +def serialize(value): + return dumps(_normalize(value), allow_nan=False) + +def deserialize(json_string): + return loads(json_string) + +def serialize_dataframe(value, key): + if not hasattr(value, 'to_csv'): + raise TypeError(f'save_dataframe() expects a pandas DataFrame or Series but received {type(value).__name__}') + if key.lower().endswith('.json'): + return value.to_json() + return value.to_csv() + +def deserialize_dataframe(serialized, key): + import pandas + from io import StringIO + if key.lower().endswith('.json'): + return pandas.read_json(StringIO(serialized)) + # to_csv() writes the index as the first column, datetimes in ISO-8601: restore a time index when it parses as one + frame = pandas.read_csv(StringIO(serialized), index_col=0) + if pandas.api.types.is_string_dtype(frame.index): + try: + frame.index = pandas.to_datetime(frame.index, format='ISO8601') + except (ValueError, TypeError): + pass + return frame +"); + _jsonDeserializeMethod = module.GetAttr("deserialize"); + _dataFrameSerializeMethod = module.GetAttr("serialize_dataframe"); + _dataFrameDeserializeMethod = module.GetAttr("deserialize_dataframe"); + // last so partial initialization is never observed + _jsonSerializeMethod = module.GetAttr("serialize"); + } + } + } +} diff --git a/Engine/Storage/LocalObjectStore.cs b/Engine/Storage/LocalObjectStore.cs index 80690b8a7531..bbe6d7ccaf81 100644 --- a/Engine/Storage/LocalObjectStore.cs +++ b/Engine/Storage/LocalObjectStore.cs @@ -292,7 +292,14 @@ public byte[] ReadBytes(string path) // Ensure we have the key, also takes care of null or improper access if (!ContainsKey(path)) { - throw new KeyNotFoundException($"Object with path '{path}' was not found in the current project. " + + // list the available keys so a near-miss (typo, wrong extension) is visible in the error itself + var keys = Keys; + var sample = string.Join(", ", keys.Take(10).Select(key => $"'{key}'")); + if (keys.Count > 10) + { + sample += ", ..."; + } + throw new KeyNotFoundException($"Object with path '{path}' was not found in the current project. Keys: [{sample}]. " + "Please use ObjectStore.ContainsKey(key) to check if an object exists before attempting to read." ); } diff --git a/Research/QuantBook.cs b/Research/QuantBook.cs index 1840312d7d24..f53cab5681b8 100644 --- a/Research/QuantBook.cs +++ b/Research/QuantBook.cs @@ -27,6 +27,7 @@ using QuantConnect.Securities.Future; using QuantConnect.Securities.Option; using QuantConnect.Statistics; +using QuantConnect.Storage; using QuantConnect.Util; using System; using System.Collections.Generic; @@ -165,7 +166,9 @@ public QuantBook() : base() StorageAccess = Config.GetValue("storage-permissions", new Packets.StoragePermissions()) }, AlgorithmMode.Research); - SetObjectStore(algorithmHandlers.ObjectStore); + SetObjectStore(_isPythonNotebook + ? new PythonObjectStore(algorithmHandlers.ObjectStore) + : new ObjectStore(algorithmHandlers.ObjectStore)); _dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider); _dataProvider = algorithmHandlers.DataProvider; diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index 6451c116bda3..4f6b6e42892a 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -19,6 +19,7 @@ using NUnit.Framework; using System.Threading; using QuantConnect.Util; +using QuantConnect.Algorithm; using QuantConnect.Logging; using QuantConnect.Packets; using QuantConnect.Storage; @@ -26,6 +27,7 @@ using System.Threading.Tasks; using System.Collections.Generic; using QuantConnect.Lean.Engine.Storage; +using System.Reflection; using Python.Runtime; namespace QuantConnect.Tests.Common.Storage @@ -36,7 +38,8 @@ public class LocalObjectStoreTests private static readonly string TestStorageRoot = $"{Directory.GetCurrentDirectory()}/{nameof(LocalObjectStoreTests)}"; private static readonly string StorageRootConfigurationValue = LocalObjectStore.DefaultObjectStore; - private ObjectStore _store; + // the Python store so the Python tests share it, the C# tests only reach the base members + private PythonObjectStore _store; private ILogHandler _logHandler; [OneTimeSetUp] @@ -44,7 +47,7 @@ public void Setup() { LocalObjectStore.DefaultObjectStore = TestStorageRoot; #pragma warning disable CA2000 - _store = new ObjectStore(new TestLocalObjectStore()); + _store = new PythonObjectStore(new TestLocalObjectStore()); #pragma warning restore CA2000 _store.Initialize(0, 0, "", new Controls() { StorageLimit = 5 * 1024 * 1024, StorageFileCount = 100 }, AlgorithmMode.Backtesting); @@ -541,6 +544,84 @@ public void ThrowsKeyNotFoundException_WhenObjectStoreDoesNotContainKey() Assert.IsTrue(error.Message.Contains("Please use ObjectStore.ContainsKey(key)")); } + [Test] + public void KeyNotFoundErrorListsTheAvailableKeys() + { + // dedicated store instance: its in-memory keys are listed first in the error + using var store = new ObjectStore(new TestLocalObjectStore()); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.IsTrue(store.SaveBytes("available_key_a.txt", new byte[] { 1 })); + Assert.IsTrue(store.SaveBytes("available_key_b.txt", new byte[] { 1 })); + + var error = Assert.Throws(() => store.ReadBytes("missing.missing")); + + Assert.IsTrue(error.Message.Contains("Keys: ["), error.Message); + Assert.IsTrue(error.Message.Contains("'available_key_a.txt'"), error.Message); + Assert.IsTrue(error.Message.Contains("'available_key_b.txt'"), error.Message); + + store.Delete("available_key_a.txt"); + store.Delete("available_key_b.txt"); + } + + [Test] + public void KeyNotFoundErrorTruncatesTheAvailableKeysListAtTen() + { + using var store = new ObjectStore(new TestLocalObjectStore()); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + for (var i = 0; i < 11; i++) + { + Assert.IsTrue(store.SaveBytes($"truncation_key_{i}.txt", new byte[] { 1 })); + } + + var error = Assert.Throws(() => store.ReadBytes("missing.missing")); + + Assert.IsTrue(error.Message.Contains(", ...]"), error.Message); + + for (var i = 0; i < 11; i++) + { + store.Delete($"truncation_key_{i}.txt"); + } + } + + [Test] + public void QuantBookUsesThePythonObjectStoreInPythonNotebooks() + { + // the notebook kernel is detected once, in the static constructor + var isPythonNotebook = typeof(QuantBook).GetField("_isPythonNotebook", BindingFlags.NonPublic | BindingFlags.Static); + isPythonNotebook.SetValue(null, true); + try + { + var quantBook = new QuantBook(); + + Assert.IsInstanceOf(quantBook.ObjectStore); + } + finally + { + isPythonNotebook.SetValue(null, false); + } + } + + [Test] + public void AlgorithmKeepsAnAlreadyWrappedStore() + { + using var pythonStore = new PythonObjectStore(new TestLocalObjectStore()); + var algorithm = new QCAlgorithm(); + + algorithm.SetObjectStore(pythonStore); + + Assert.AreSame(pythonStore, algorithm.ObjectStore); + } + + [Test] + public void SaveTextAndReadTextAreAliasesOfSaveAndRead() + { + Assert.IsTrue(_store.SaveText("save_text_alias.txt", "abc")); + Assert.AreEqual("abc", _store.Read("save_text_alias.txt")); + Assert.AreEqual("abc", _store.ReadText("save_text_alias.txt")); + + _store.Delete("save_text_alias.txt"); + } + [TestCase("my_key", "./LocalObjectStoreTests/my_key")] [TestCase("test/123", "./LocalObjectStoreTests/test/123")] [TestCase("**abc**", null)] @@ -1016,6 +1097,350 @@ def add_data(object_store): } } + [Test] + public void PythonSaveJsonAndReadJsonRoundTripTolerantTypes() + { + RunPythonTest(@" +def test(object_store): + symbol = Symbol.create('SPY', SecurityType.EQUITY, Market.USA) + + # datetime/date/Decimal/Symbol values and non-string dictionary keys, which json.dumps rejects + object_store.save_json('helpers/log.json', { + 'time': datetime(2024, 1, 2, 3, 4, 5), + 'date': datetime(2024, 1, 1).date(), + 'qty': Decimal('1.5'), + 'symbol': symbol, + 'fills': [{symbol: 10}] + }) + data = object_store.read_json('helpers/log.json') + # a Symbol value is written like the C# SaveJson does, a Symbol key as its string like a .NET dictionary key + assert data == { + 'time': '2024-01-02T03:04:05', + 'date': '2024-01-01', + 'qty': 1.5, + 'symbol': {'value': 'SPY', 'id': str(symbol.id), 'permtick': 'SPY'}, + 'fills': [{str(symbol): 10}] + }, data + # both forms resolve back to the Symbol through the symbol cache, which is what QCAlgorithm.symbol(str) does + assert SymbolCache.get_symbol(data['symbol']['id']) == symbol + assert SymbolCache.get_symbol(next(iter(data['fills'][0]))) == symbol + + object_store.delete('helpers/log.json') +"); + } + + [Test] + public void PythonSaveJsonSerializesDotNetObjectsLikeTheCSharpOverload() + { + RunPythonTest(@" +from System.Collections.Generic import List, Dictionary +import Newtonsoft.Json as newtonsoft + +def test(object_store): + numbers = List[int]([1, 2]) + weights = Dictionary[str, float]() + weights['a'] = 1.5 + point = IndicatorDataPoint(datetime(2020, 1, 1), 5) + + object_store.save_json('helpers/dotnet.json', {'numbers': numbers, 'weights': weights, 'point': point}) + data = object_store.read_json('helpers/dotnet.json') + assert data == { + 'numbers': [1, 2], + 'weights': {'a': 1.5}, + 'point': json.loads(newtonsoft.JsonConvert.serialize_object(point)) + }, data + + object_store.delete('helpers/dotnet.json') +"); + } + + [Test] + public void PythonSaveJsonConvertsNumpyArraysAndScalars() + { + RunPythonTest(@" +import numpy as np + +def test(object_store): + object_store.save_json('helpers/numpy.json', { + 'weights': np.array([1.5, 2.5]), + 'grid': np.array([[1, 2], [3, 4]]), + 'count': np.int64(3), + 'ratio': np.float32(0.5) + }) + data = object_store.read_json('helpers/numpy.json') + assert data == {'weights': [1.5, 2.5], 'grid': [[1, 2], [3, 4]], 'count': 3, 'ratio': 0.5}, data + + object_store.delete('helpers/numpy.json') +"); + } + + [Test] + public void PythonSaveJsonWritesNanAndInfinityAsNull() + { + RunPythonTest(@" +import numpy as np + +def test(object_store): + object_store.save_json('helpers/nan.json', { + 'pnl': float('nan'), + 'ratio': float('inf'), + 'drawdown': -float('inf'), + 'series': np.array([1.0, np.nan]), + 'scalar': np.float64('nan') + }) + text = object_store.read('helpers/nan.json') + assert 'NaN' not in text and 'Infinity' not in text, text + data = object_store.read_json('helpers/nan.json') + assert data == {'pnl': None, 'ratio': None, 'drawdown': None, 'series': [1.0, None], 'scalar': None}, data + + object_store.delete('helpers/nan.json') +"); + } + + [Test] + public void PythonSaveJsonWritesNumpyDatetimesAsIso8601AndNatAsNull() + { + RunPythonTest(@" +import numpy as np + +def test(object_store): + object_store.save_json('helpers/datetime64.json', { + 'index': pd.DatetimeIndex([pd.Timestamp('2020-01-01'), pd.Timestamp('2020-01-02 03:04:05.000000006')]).values, + 'scalar': np.datetime64('2020-01-01T00:00:00.000000000'), + 'day': np.datetime64('2020-01-01'), + 'timestamp': pd.Timestamp('2020-01-01 10:00'), + 'nat': pd.NaT, + 'nat64': np.datetime64('NaT'), + 'with_nat': pd.to_datetime(['2020-01-01', None]).values, + 'grid': np.array([['2020-01-01', '2020-01-02']], dtype='datetime64[D]') + }) + data = object_store.read_json('helpers/datetime64.json') + assert data == { + 'index': ['2020-01-01T00:00:00', '2020-01-02T03:04:05.000000006'], + 'scalar': '2020-01-01T00:00:00', + 'day': '2020-01-01T00:00:00', + 'timestamp': '2020-01-01T10:00:00', + 'nat': None, + 'nat64': None, + 'with_nat': ['2020-01-01T00:00:00', None], + 'grid': [['2020-01-01T00:00:00', '2020-01-02T00:00:00']] + }, data + + object_store.delete('helpers/datetime64.json') +"); + } + + [Test] + public void PythonSaveJsonWritesTimedeltasAsIso8601Durations() + { + RunPythonTest(@" +import numpy as np +from datetime import timedelta + +def test(object_store): + object_store.save_json('helpers/timedelta.json', { + 'holding': timedelta(days=1, hours=2), + 'latency': np.timedelta64(1500, 'ms'), + 'spans': (pd.to_datetime(['2020-01-02', '2020-01-03']) - pd.Timestamp('2020-01-01')).values, + 'pandas': pd.Timedelta(seconds=1.5), + 'nat': np.timedelta64('NaT') + }) + data = object_store.read_json('helpers/timedelta.json') + assert data == { + 'holding': 'P1DT2H0M0S', + 'latency': 'P0DT0H0M1.5S', + 'spans': ['P1DT0H0M0S', 'P2DT0H0M0S'], + 'pandas': 'P0DT0H0M1.5S', + 'nat': None + }, data + assert pd.Timedelta(data['holding']) == timedelta(days=1, hours=2) + + object_store.delete('helpers/timedelta.json') +"); + } + + [Test] + public void PythonSaveJsonRejectsNanDictionaryKeysWithTypeError() + { + RunPythonTest(@" +def test(object_store): + error = None + try: + object_store.save_json('helpers/nan_key.json', {float('nan'): 1}) + except TypeError as e: + error = str(e) + assert error is not None and 'nan' in error and 'key' in error, error + assert not object_store.contains_key('helpers/nan_key.json') +"); + } + + [Test] + public void PythonSaveJsonRejectsPandasObjectsPointingToSaveDataframe() + { + RunPythonTest(@" +def test(object_store): + for value in [pd.DataFrame({'a': [1, 2]}), pd.Series([1, 2])]: + error = None + try: + object_store.save_json('helpers/pandas.json', {'data': value}) + except TypeError as e: + error = str(e) + assert error is not None and 'save_dataframe' in error and type(value).__name__ in error, error + assert not object_store.contains_key('helpers/pandas.json') +"); + } + + [Test] + public void PythonSaveJsonRejectsUnsupportedTypesNamingTheType() + { + RunPythonTest(@" +class TradeRecord: + pass + +def test(object_store): + for value in [TradeRecord(), object(), (x for x in [1])]: + error = None + try: + object_store.save_json('helpers/unsupported.json', {'data': value}) + except TypeError as e: + error = str(e) + assert error is not None and type(value).__name__ in error, error + assert not object_store.contains_key('helpers/unsupported.json') +"); + } + + [Test] + public void PythonSaveJsonRoundTripsTopLevelContainersAndRejectsTopLevelScalars() + { + RunPythonTest(@" +def test(object_store): + failures = [] + try: + for value in [None, [1, 'a'], (1, 2)]: + expected = list(value) if isinstance(value, tuple) else value + object_store.save_json('helpers/container.json', value) + data = object_store.read_json('helpers/container.json') + if data != expected: + failures.append(f'{value!r} -> {data!r}') + + # a top-level str/int/float/bool does not bind to the PyObject overload: the error names the mismatch + for value in ['text', 42, 1.5, True]: + try: + object_store.save_json('helpers/scalar.json', value) + failures.append(f'{value!r} was accepted') + except TypeError as e: + if 'No method matches' not in str(e): + failures.append(f'{value!r} -> {e}') + if object_store.contains_key('helpers/scalar.json'): + failures.append('a rejected scalar was saved') + finally: + for key in ['helpers/container.json', 'helpers/scalar.json']: + if object_store.contains_key(key): + object_store.delete(key) + assert not failures, failures +"); + } + + [Test] + public void PythonReadJsonThrowsOnMissingKey() + { + RunPythonTest(@" +def test(object_store): + assert_raises_key_not_found(lambda: object_store.read_json('helpers/missing.json')) +"); + } + + [Test] + public void PythonSaveTextAndReadTextAreAliasesOfSaveAndRead() + { + RunPythonTest(@" +def test(object_store): + object_store.save_text('helpers/report.txt', 'hello') + assert object_store.read('helpers/report.txt') == 'hello' + assert object_store.read_text('helpers/report.txt') == 'hello' + + object_store.delete('helpers/report.txt') +"); + } + + [Test] + public void PythonSaveDataframeAndReadDataframeRoundTripAsCsv() + { + RunPythonTest(@" +def test(object_store): + frame = pd.DataFrame({'a': [1, 2], 'b': [3.5, 4.5]}) + object_store.save_dataframe('helpers/frame.csv', frame) + assert 'a,b' in object_store.read('helpers/frame.csv') + pd.testing.assert_frame_equal(object_store.read_dataframe('helpers/frame.csv'), frame) + + object_store.delete('helpers/frame.csv') +"); + } + + [Test] + public void PythonSaveDataframeAndReadDataframeRoundTripAsJson() + { + RunPythonTest(@" +def test(object_store): + frame = pd.DataFrame({'a': [1, 2], 'b': [3.5, 4.5]}) + object_store.save_dataframe('helpers/frame.json', frame) + assert object_store.read('helpers/frame.json') == frame.to_json() + pd.testing.assert_frame_equal(object_store.read_dataframe('helpers/frame.json'), frame) + + object_store.delete('helpers/frame.json') +"); + } + + [Test] + public void PythonReadDataframeRestoresATimeIndex() + { + RunPythonTest(@" +def test(object_store): + frame = pd.DataFrame({'close': [167.42, 168.0]}, index=pd.to_datetime(['2013-10-10', '2013-10-11'])) + object_store.save_dataframe('helpers/history.csv', frame) + pd.testing.assert_frame_equal(object_store.read_dataframe('helpers/history.csv'), frame) + + object_store.delete('helpers/history.csv') +"); + } + + [Test] + public void PythonReadDataframeThrowsOnMissingKey() + { + RunPythonTest(@" +def test(object_store): + assert_raises_key_not_found(lambda: object_store.read_dataframe('helpers/missing.csv')) +"); + } + + /// + /// Runs the 'test(object_store)' function of the given Python source against the test store + /// + private void RunPythonTest(string source) + { + using (Py.GIL()) + { + var testModule = PyModule.FromString("TestObjectStoreHelpers", @" +from AlgorithmImports import * +from System.Collections.Generic import KeyNotFoundException +from datetime import datetime +from decimal import Decimal +import json +import pandas as pd + +def assert_raises_key_not_found(action): + try: + action() + except KeyNotFoundException: + return + raise AssertionError('Expected a KeyNotFoundException') +" + source); + + dynamic test = testModule.GetAttr("test"); + Assert.DoesNotThrow(() => test(_store)); + } + } + private static void DummyMachineLearning(string outputFile, string content) { try diff --git a/Tests/Python/AlgorithmPythonWrapperTests.cs b/Tests/Python/AlgorithmPythonWrapperTests.cs index f2b406c9482c..2dcab23c97a5 100644 --- a/Tests/Python/AlgorithmPythonWrapperTests.cs +++ b/Tests/Python/AlgorithmPythonWrapperTests.cs @@ -21,6 +21,9 @@ using System.Collections.Generic; using System.IO; using QuantConnect.Orders; +using QuantConnect.Lean.Engine.Storage; +using QuantConnect.Packets; +using QuantConnect.Storage; namespace QuantConnect.Tests.Python { @@ -149,6 +152,27 @@ public void CallOnEndOfDayExceptionWithParameter() } } + [Test] + public void PythonAlgorithmGetsAPythonObjectStore() + { + using (Py.GIL()) + { + var algorithm = GetAlgorithm("def on_end_of_algorithm(self): self.object_store.save_json('trade_log.json', {'a': 1})"); + using var localStore = new LocalObjectStore(); + localStore.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + + algorithm.SetObjectStore(localStore); + + Assert.IsInstanceOf(algorithm.ObjectStore); + // the Python helpers are reachable through self.object_store + algorithm.OnEndOfAlgorithm(); + Assert.Null(algorithm.RunTimeError); + Assert.AreEqual("{\"a\": 1}", algorithm.ObjectStore.Read("trade_log.json")); + + localStore.Delete("trade_log.json"); + } + } + private AlgorithmPythonWrapper GetAlgorithm(string code) { code = $"{_baseCode}{Environment.NewLine} {code}";