Improve ObjectStore ergonomics for end-of-run persistence

Algorithm.CSharp/ObjectStoreHelpersRegressionAlgorithm.cs

@@ -0,0 +1,134 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting the ObjectStore helpers: <see cref="Storage.ObjectStore.SaveText"/> and
+    /// <see cref="Storage.ObjectStore.ReadText"/> are aliases of Save and Read, and reading a missing key lists the
+    /// available keys in the error. The Python version of this algorithm additionally covers the tolerant
+    /// save_json/read_json and save_dataframe/read_dataframe helpers
+    /// </summary>
+    public class ObjectStoreHelpersRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        /// <summary>
+        /// Initialize the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetStartDate(2013, 10, 7);
+            SetEndDate(2013, 10, 11);
+
+            SetBenchmark(x => 0);
+
+            AddEquity("SPY", Resolution.Daily);
+        }
+
+        /// <summary>
+        /// End of algorithm run event handler. This method is called at the end of a backtest or live trading operation.
+        /// </summary>
+        public override void OnEndOfAlgorithm()
+        {
+            // SaveText and ReadText are aliases of Save and Read
+            if (!ObjectStore.SaveText("helpers_report.txt", "The strategy went up") ||
+                ObjectStore.Read("helpers_report.txt") != "The strategy went up" ||
+                ObjectStore.ReadText("helpers_report.txt") != "The strategy went up")
+            {
+                throw new RegressionTestException("Expected the SaveText/ReadText round trip to succeed");
+            }
+
+            // reading a missing key lists the available keys in the error
+            var errorMessage = string.Empty;
+            try
+            {
+                ObjectStore.ReadBytes("helpers_missing.json");
+            }
+            catch (KeyNotFoundException exception)
+            {
+                errorMessage = exception.Message;
+            }
+            if (!errorMessage.Contains("Keys: [") || !errorMessage.Contains("'helpers_report.txt'"))
+            {
+                throw new RegressionTestException($"Expected the available keys in the missing key error, got: '{errorMessage}'");
+            }
+
+            ObjectStore.Delete("helpers_report.txt");
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public List<Language> Languages { get; } = new() { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public long DataPoints => 6;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public int AlgorithmHistoryDataPoints => 0;
+
+        /// <summary>
+        /// Final status of the algorithm
+        /// </summary>
+        public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Orders", "0"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Start Equity", "100000"},
+            {"End Equity", "100000"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Sortino Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "0"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$0.00"},
+            {"Estimated Strategy Capacity", "$0"},
+            {"Lowest Capacity Asset", ""},
+            {"Portfolio Turnover", "0%"},
+            {"Drawdown Recovery", "0"},
+            {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"}
+        };
+    }
+}

Algorithm.Python/ObjectStoreHelpersRegressionAlgorithm.py

@@ -0,0 +1,109 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+from System.Collections.Generic import KeyNotFoundException, List
+from decimal import Decimal
+
+### <summary>
+### Regression algorithm asserting the ObjectStore helpers: save_text and read_text are aliases of save and read,
+### reading a missing key lists the available keys in the error, and the tolerant save_json/read_json and
+### save_dataframe/read_dataframe helpers handle datetime/date/Decimal/Symbol and data frames out of the box.
+### </summary>
+class ObjectStoreHelpersRegressionAlgorithm(QCAlgorithm):
+
+    def initialize(self):
+        self.set_start_date(2013, 10, 7)
+        self.set_end_date(2013, 10, 11)
+
+        self.set_benchmark(lambda x: 0)
+
+        self.spy = self.add_equity("SPY", Resolution.DAILY).symbol
+
+    def on_end_of_algorithm(self):
+        # save_json tolerates datetime/date/Decimal/numpy/NaN values, .NET objects and non-string dictionary keys
+        trade_log = {
+            "time": datetime(2013, 10, 11, 16, 0, 0),
+            "date": date(2013, 10, 11),
+            "qty": Decimal("1.5"),
+            "symbol": self.spy,
+            "fills": [{"price": Decimal("167.42"), self.spy: 10}],
+            "fill_quantities": List[int]([10, -10]),
+            "returns": np.array([0.01, np.nan]),
+            "bar_times": pd.DatetimeIndex([datetime(2013, 10, 10), datetime(2013, 10, 11)]).values
+        }
+        if not self.object_store.save_json("helpers/trade_log.json", trade_log):
+            raise AssertionError("Expected save_json to succeed")
+        data = self.object_store.read_json("helpers/trade_log.json")
+        expected = {
+            "time": "2013-10-11T16:00:00",
+            "date": "2013-10-11",
+            "qty": 1.5,
+            "symbol": {"value": "SPY", "id": str(self.spy.id), "permtick": "SPY"},
+            "fills": [{"price": 167.42, "SPY": 10}],
+            "fill_quantities": [10, -10],
+            "returns": [0.01, None],
+            "bar_times": ["2013-10-10T00:00:00", "2013-10-11T00:00:00"]
+        }
+        if data != expected:
+            raise AssertionError(f"Unexpected read_json round trip result: {data}")
+        # the symbol id and the ticker key both resolve back to the added security through the symbol cache
+        if self.symbol(data["symbol"]["id"]) != self.spy or self.symbol("SPY") != self.spy:
+            raise AssertionError(f"Expected the stored symbol to resolve to {self.spy}, got: {data['symbol']}")
+
+        # save_json refuses what it cannot serialize faithfully instead of storing its string
+        error = None
+        try:
+            self.object_store.save_json("helpers/rejected.json", {"history": pd.Series([1.0, 2.0])})
+        except TypeError as e:
+            error = str(e)
+        if error is None or "save_dataframe" not in error:
+            raise AssertionError(f"Expected save_json to reject a pandas Series pointing to save_dataframe, got: '{error}'")
+
+        # read_json throws on a missing key like every other read method
+        error = None
+        try:
+            self.object_store.read_json("helpers/missing.json")
+        except KeyNotFoundException as e:
+            error = str(e)
+        if error is None:
+            raise AssertionError("Expected read_json of a missing key to throw")
+
+        # save_dataframe stores a pandas DataFrame as CSV and read_dataframe restores it, time index included
+        frame = pd.DataFrame({"close": [167.42, 168.0]}, index=pd.to_datetime(["2013-10-10", "2013-10-11"]))
+        if not self.object_store.save_dataframe("helpers/history.csv", frame):
+            raise AssertionError("Expected save_dataframe to succeed")
+        csv = self.object_store.read("helpers/history.csv")
+        if "close" not in csv or "2013-10-10" not in csv:
+            raise AssertionError(f"Unexpected save_dataframe content: {csv}")
+        restored = self.object_store.read_dataframe("helpers/history.csv")
+        if not restored.equals(frame):
+            raise AssertionError(f"Unexpected read_dataframe round trip result: {restored}")
+
+        # save_text/read_text are aliases of save/read
+        if not self.object_store.save_text("helpers/report.txt", "The strategy went up"):
+            raise AssertionError("Expected save_text to succeed")
+        if self.object_store.read_text("helpers/report.txt") != "The strategy went up":
+            raise AssertionError("Expected the save_text/read_text round trip to succeed")
+
+        # reading a missing key lists the available keys in the error
+        error = None
+        try:
+            self.object_store.read_bytes("helpers/missing.json")
+        except BaseException as e:
+            error = str(e)
+        if error is None or "Keys: [" not in error or "'helpers/report.txt'" not in error:
+            raise AssertionError(f"Expected the available keys in the missing key error, got: '{error}'")
+
+        for key in ["helpers/trade_log.json", "helpers/history.csv", "helpers/report.txt"]:
+            self.object_store.delete(key)

Algorithm/QCAlgorithm.cs

@@ -3308,7 +3308,8 @@ public void SetApi(IApi api)
         [DocumentationAttribute(MachineLearning)]
         public void SetObjectStore(IObjectStore objectStore)
         {
-            ObjectStore = new ObjectStore(objectStore);
+            // an already wrapped store, like the Python one, is used as is
+            ObjectStore = objectStore as ObjectStore ?? new ObjectStore(objectStore);
         }
 
         /// <summary>

AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs

@@ -1231,7 +1231,7 @@ public void SetCurrentSlice(Slice slice)
         /// Sets the object store
         /// </summary>
         /// <param name="objectStore">The object store</param>
-        public void SetObjectStore(IObjectStore objectStore) => _baseAlgorithm.SetObjectStore(objectStore);
+        public void SetObjectStore(IObjectStore objectStore) => _baseAlgorithm.SetObjectStore(new PythonObjectStore(objectStore));
 
         /// <summary>
         /// Determines if the Symbol is shortable at the brokerage

Common/Storage/ObjectStore.cs

@@ -159,6 +159,17 @@ public string ReadString(string path, Encoding encoding = null)
             return Read(path, encoding);
         }
 
+        /// <summary>
+        /// Returns the object data for the specified path as a string. Alias of <see cref="Read(string, Encoding)"/>
+        /// </summary>
+        /// <param name="path">The object path</param>
+        /// <param name="encoding">The string encoding used, <see cref="Encoding.UTF8"/> by default</param>
+        /// <returns>A string containing the data</returns>
+        public string ReadText(string path, Encoding encoding = null)
+        {
+            return Read(path, encoding);
+        }
+
         /// <summary>
         /// Returns the JSON deserialized object data for the specified path
         /// </summary>
@@ -239,6 +250,19 @@ public bool SaveString(string path, string text, Encoding encoding = null)
             return _store.SaveBytes(path, encoding.GetBytes(text));
         }
 
+        /// <summary>
+        /// Saves the object data in text format for the specified path.
+        /// Alias of <see cref="Save(string, string, Encoding)"/>
+        /// </summary>
+        /// <param name="path">The object path</param>
+        /// <param name="text">The string object to be saved</param>
+        /// <param name="encoding">The string encoding used, <see cref="Encoding.UTF8"/> by default</param>
+        /// <returns>True if the object was saved successfully</returns>
+        public bool SaveText(string path, string text, Encoding encoding = null)
+        {
+            return Save(path, text, encoding);
+        }
+
         /// <summary>
         /// Saves the object data in JSON format for the specified path
         /// </summary>
@@ -299,4 +323,4 @@ public void Dispose()
             _store.Dispose();
         }
     }
-}
+}