diff --git a/Algorithm.CSharp/ObjectStoreKeySanitizationRegressionAlgorithm.cs b/Algorithm.CSharp/ObjectStoreKeySanitizationRegressionAlgorithm.cs new file mode 100644 index 000000000000..0557fae700f9 --- /dev/null +++ b/Algorithm.CSharp/ObjectStoreKeySanitizationRegressionAlgorithm.cs @@ -0,0 +1,143 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Collections.Generic; +using System.Linq; +using QuantConnect.Interfaces; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Regression algorithm asserting the ObjectStore key rules: an unsupported key is sanitized and used on every + /// save and read, and a save through it replaces the object stored under its sanitized name + /// + public class ObjectStoreKeySanitizationRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + /// + /// Initialize the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. + /// + public override void Initialize() + { + SetStartDate(2013, 10, 7); + SetEndDate(2013, 10, 11); + + SetBenchmark(x => 0); + + AddEquity("SPY", Resolution.Daily); + } + + /// + /// End of algorithm run event handler. This method is called at the end of a backtest or live trading operation. + /// + public override void OnEndOfAlgorithm() + { + // an unsupported key, e.g. built from a user-facing name, is sanitized and used on every save and read + const string invalidKey = "trade_log_ai_hardware_&_cloud.csv"; + const string sanitizedKey = "trade_log_ai_hardware___cloud.csv"; + if (!ObjectStore.Save(invalidKey, "a,b\n1,2")) + { + throw new RegressionTestException("Expected the unsupported key to be sanitized and saved"); + } + var keys = ObjectStore.Keys.ToList(); + if (!keys.Contains(sanitizedKey) || !keys.Contains(invalidKey)) + { + throw new RegressionTestException($"Expected the keys to hold '{sanitizedKey}' and '{invalidKey}', keys: {string.Join(", ", keys)}"); + } + if (!ObjectStore.ContainsKey(invalidKey) || !ObjectStore.Keys.Contains(invalidKey) + || ObjectStore.Read(invalidKey) != "a,b\n1,2" || ObjectStore.Read(sanitizedKey) != "a,b\n1,2") + { + throw new RegressionTestException("Expected the object to be read through both the original and the sanitized key"); + } + + // a later save through the same key updates the object + if (!ObjectStore.Save(invalidKey, "a,b\n3,4") || ObjectStore.Read(sanitizedKey) != "a,b\n3,4") + { + throw new RegressionTestException("Expected the second save through the unsupported key to update the object"); + } + + // a save through the unsupported key replaces the object under its sanitized name, whoever wrote it + if (!ObjectStore.Save(sanitizedKey, "c,d\n5,6") || !ObjectStore.Save(invalidKey, "e,f\n7,8") || ObjectStore.Read(sanitizedKey) != "e,f\n7,8") + { + throw new RegressionTestException("Expected the save through the unsupported key to replace the object under its sanitized name"); + } + + ObjectStore.Delete(sanitizedKey); + if (ObjectStore.ContainsKey(invalidKey) || ObjectStore.Keys.ToList().Contains(invalidKey)) + { + throw new RegressionTestException("Expected the delete of the sanitized key to remove the object the unsupported key reads"); + } + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public List Languages { get; } = new() { Language.CSharp, Language.Python }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public long DataPoints => 6; + + /// + /// Data Points count of the algorithm history + /// + public int AlgorithmHistoryDataPoints => 0; + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "0"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "0%"}, + {"Drawdown", "0%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "100000"}, + {"Net Profit", "0%"}, + {"Sharpe Ratio", "0"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "0%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "0"}, + {"Beta", "0"}, + {"Annual Standard Deviation", "0"}, + {"Annual Variance", "0"}, + {"Information Ratio", "0"}, + {"Tracking Error", "0"}, + {"Treynor Ratio", "0"}, + {"Total Fees", "$0.00"}, + {"Estimated Strategy Capacity", "$0"}, + {"Lowest Capacity Asset", ""}, + {"Portfolio Turnover", "0%"}, + {"Drawdown Recovery", "0"}, + {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"} + }; + } +} diff --git a/Algorithm.Python/ObjectStoreKeySanitizationRegressionAlgorithm.py b/Algorithm.Python/ObjectStoreKeySanitizationRegressionAlgorithm.py new file mode 100644 index 000000000000..17884f04ef9e --- /dev/null +++ b/Algorithm.Python/ObjectStoreKeySanitizationRegressionAlgorithm.py @@ -0,0 +1,55 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from AlgorithmImports import * + +### +### Regression algorithm asserting the ObjectStore key rules: an unsupported key is sanitized and used on every +### save and read, and a save through it replaces the object stored under its sanitized name +### +class ObjectStoreKeySanitizationRegressionAlgorithm(QCAlgorithm): + + def initialize(self): + self.set_start_date(2013, 10, 7) + self.set_end_date(2013, 10, 11) + + self.set_benchmark(lambda x: 0) + + self.add_equity("SPY", Resolution.DAILY) + + def on_end_of_algorithm(self): + # an unsupported key, e.g. built from a user-facing name, is sanitized and used on every save and read + invalid_key = "trade_log_ai_hardware_&_cloud.csv" + sanitized_key = "trade_log_ai_hardware___cloud.csv" + if not self.object_store.save(invalid_key, "a,b\n1,2"): + raise AssertionError("Expected the unsupported key to be sanitized and saved") + keys = list(self.object_store.keys) + if sanitized_key not in keys or invalid_key not in keys: + raise AssertionError(f"Expected the keys to hold '{sanitized_key}' and '{invalid_key}', keys: {keys}") + if not self.object_store.contains_key(invalid_key) or invalid_key not in self.object_store.keys \ + or self.object_store.read(invalid_key) != "a,b\n1,2" \ + or self.object_store.read(sanitized_key) != "a,b\n1,2": + raise AssertionError("Expected the object to be read through both the original and the sanitized key") + + # a later save through the same key updates the object + if not self.object_store.save(invalid_key, "a,b\n3,4") or self.object_store.read(sanitized_key) != "a,b\n3,4": + raise AssertionError("Expected the second save through the unsupported key to update the object") + + # a save through the unsupported key replaces the object under its sanitized name, whoever wrote it + if not self.object_store.save(sanitized_key, "c,d\n5,6") or not self.object_store.save(invalid_key, "e,f\n7,8") \ + or self.object_store.read(sanitized_key) != "e,f\n7,8": + raise AssertionError("Expected the save through the unsupported key to replace the object under its sanitized name") + + self.object_store.delete(sanitized_key) + if self.object_store.contains_key(invalid_key) or invalid_key in list(self.object_store.keys): + raise AssertionError("Expected the delete of the sanitized key to remove the object the unsupported key reads") diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index 280dee74a02d..20240dae96b8 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -3321,7 +3321,7 @@ public void SetApi(IApi api) public void SetObjectStore(IObjectStore objectStore) { // an already wrapped store, like the Python one, is used as is - ObjectStore = objectStore as ObjectStore ?? new ObjectStore(objectStore); + ObjectStore = objectStore as ObjectStore ?? new ObjectStore(objectStore, Debug); } /// diff --git a/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs b/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs index 2fafbc4f675c..526906505687 100644 --- a/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs +++ b/AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs @@ -1231,7 +1231,7 @@ public void SetCurrentSlice(Slice slice) /// Sets the object store /// /// The object store - public void SetObjectStore(IObjectStore objectStore) => _baseAlgorithm.SetObjectStore(new PythonObjectStore(objectStore)); + public void SetObjectStore(IObjectStore objectStore) => _baseAlgorithm.SetObjectStore(new PythonObjectStore(objectStore, _baseAlgorithm.Debug)); /// /// Determines if the Symbol is shortable at the brokerage diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index a610c1d00752..74b6af0963fb 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -15,21 +15,41 @@ using System; using System.Collections; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; +using System.Threading; +using System.Text.RegularExpressions; using System.Xml.Serialization; using Newtonsoft.Json; using QuantConnect.Interfaces; +using QuantConnect.Logging; using QuantConnect.Packets; namespace QuantConnect.Storage { /// - /// Helper class for easier access to methods + /// Helper class for easier access to methods. A key that does not follow the key + /// format (english letters, numbers, spaces and the characters '.', '/', '\', '_', '#', '-', '$' and '=', never + /// two '.' in a row, e.g. 'folder/trade_log-2024.v2.csv') is saved under its sanitized version, then found and + /// listed by either name, with a warning the first time it happens. An object already stored under an unsupported name + /// is read as is until a save through that name creates the sanitized one /// public class ObjectStore : IObjectStore { + // The single source of the key charset rules, also enforced by the LocalObjectStore implementation + private static readonly Regex UnsupportedKeyCharactersRegex = new(@"[^a-zA-Z0-9\\/_#\-\$= .]", RegexOptions.Compiled); + // '..' is never allowed: it would leave the storage root + private static readonly Regex ConsecutiveDotsRegex = new(@"\.{2,}", RegexOptions.Compiled); + + private readonly IObjectStore _store; + private readonly Action _warn; + // The unsupported keys saved in this run and the sanitized name each one is stored under + private readonly ConcurrentDictionary _sanitizedKeys = new(); + private int _warned; + /// /// Gets the maximum storage limit in bytes /// @@ -49,15 +69,15 @@ public event EventHandler ErrorRaised remove { _store.ErrorRaised -= value; } } - private readonly IObjectStore _store; - /// /// Initializes a new instance of the class /// /// The instance to wrap - public ObjectStore(IObjectStore store) + /// Receives the warnings about sanitized keys, the engine log by default + public ObjectStore(IObjectStore store, Action warn = null) { _store = store; + _warn = warn ?? (message => Log.Trace(message)); } /// @@ -74,9 +94,11 @@ public void Initialize(int userId, int projectId, string userToken, Controls con } /// - /// Returns the file paths present in the object store. This is specially useful not to load the object store into memory + /// Returns the file paths present in the object store. This is specially useful not to load the object store into memory. + /// An unsupported key saved in this run is listed next to the sanitized name its object is stored under. + /// The store is listed on first use, and Contains finds a key the way does, without listing it /// - public ICollection Keys => _store.Keys; + public ICollection Keys => new KeyCollection(ListKeys, ContainsKey); /// /// Will clear the object store state cache. This is useful when the object store is used concurrently by nodes which want to share information @@ -90,7 +112,7 @@ public void Initialize(int userId, int projectId, string userToken, Controls con /// True if the key was found public bool ContainsKey(string path) { - return _store.ContainsKey(path); + return _store.ContainsKey(KeyToRead(path)); } /// @@ -100,18 +122,18 @@ public bool ContainsKey(string path) /// A byte array containing the data public byte[] ReadBytes(string path) { - return _store.ReadBytes(path); + return _store.ReadBytes(KeyToRead(path)); } /// /// Saves the object data for the specified path /// - /// The object path + /// The object key: english letters, numbers, spaces and the characters '.', '/', '\', '_', '#', '-', '$' and '=', never '..', e.g. 'folder/trade_log-2024.v2.csv'. Other characters are replaced by '_' /// The object data /// True if the save operation was successful public bool SaveBytes(string path, byte[] contents) { - return _store.SaveBytes(path, contents); + return _store.SaveBytes(KeyToSave(path), contents); } /// @@ -121,7 +143,7 @@ public bool SaveBytes(string path, byte[] contents) /// True if the delete operation was successful public bool Delete(string path) { - return _store.Delete(path); + return _store.Delete(KeyToRead(path)); } /// @@ -131,7 +153,7 @@ public bool Delete(string path) /// The path for the file public string GetFilePath(string path) { - return _store.GetFilePath(path); + return _store.GetFilePath(KeyToRead(path)); } /// @@ -144,7 +166,7 @@ public string Read(string path, Encoding encoding = null) { encoding = encoding ?? Encoding.UTF8; - var data = _store.ReadBytes(path); + var data = ReadBytes(path); return data != null ? encoding.GetString(data) : null; } @@ -208,7 +230,7 @@ public T ReadXml(string path, Encoding encoding = null) /// Saves the data from a local file path associated with the specified path /// /// If the file does not exist it will throw an exception - /// The object path + /// The object key: english letters, numbers, spaces and the characters '.', '/', '\', '_', '#', '-', '$' and '=', never '..', e.g. 'folder/trade_log-2024.v2.csv'. Other characters are replaced by '_' /// True if the object was saved successfully public bool Save(string path) { @@ -220,26 +242,26 @@ public bool Save(string path) } var bytes = File.ReadAllBytes(filePath); - return _store.SaveBytes(path, bytes); + return SaveBytes(path, bytes); } /// /// Saves the object data in text format for the specified path /// - /// The object path + /// The object key: english letters, numbers, spaces and the characters '.', '/', '\', '_', '#', '-', '$' and '=', never '..', e.g. 'folder/trade_log-2024.v2.csv'. Other characters are replaced by '_' /// The string object to be saved /// The string encoding used, by default /// True if the object was saved successfully public bool Save(string path, string text, Encoding encoding = null) { encoding ??= Encoding.UTF8; - return _store.SaveBytes(path, encoding.GetBytes(text)); + return SaveBytes(path, encoding.GetBytes(text)); } /// /// Saves the object data in text format for the specified path /// - /// The object path + /// The object key: english letters, numbers, spaces and the characters '.', '/', '\', '_', '#', '-', '$' and '=', never '..', e.g. 'folder/trade_log-2024.v2.csv'. Other characters are replaced by '_' /// The string object to be saved /// The string encoding used /// True if the object was saved successfully @@ -247,14 +269,14 @@ public bool SaveString(string path, string text, Encoding encoding = null) { encoding = encoding ?? Encoding.UTF8; - return _store.SaveBytes(path, encoding.GetBytes(text)); + return SaveBytes(path, encoding.GetBytes(text)); } /// /// Saves the object data in text format for the specified path. /// Alias of /// - /// The object path + /// The object key: english letters, numbers, spaces and the characters '.', '/', '\', '_', '#', '-', '$' and '=', never '..', e.g. 'folder/trade_log-2024.v2.csv'. Other characters are replaced by '_' /// The string object to be saved /// The string encoding used, by default /// True if the object was saved successfully @@ -266,7 +288,7 @@ public bool SaveText(string path, string text, Encoding encoding = null) /// /// Saves the object data in JSON format for the specified path /// - /// The object path + /// The object key: english letters, numbers, spaces and the characters '.', '/', '\', '_', '#', '-', '$' and '=', never '..', e.g. 'folder/trade_log-2024.v2.csv'. Other characters are replaced by '_' /// The object to be saved /// The string encoding used /// The settings used by the JSON serializer @@ -282,7 +304,7 @@ public bool SaveJson(string path, T obj, Encoding encoding = null, JsonSerial /// /// Saves the object data in XML format for the specified path /// - /// The object path + /// The object key: english letters, numbers, spaces and the characters '.', '/', '\', '_', '#', '-', '$' and '=', never '..', e.g. 'folder/trade_log-2024.v2.csv'. Other characters are replaced by '_' /// The object to be saved /// The string encoding used /// True if the object was saved successfully @@ -322,5 +344,226 @@ public void Dispose() { _store.Dispose(); } + + /// + /// Throws if the given key does not follow the key format, stating every problem found and the rules. + /// The last line of defense of the store implementations, as this class only hands them supported keys + /// + /// The object store key to validate + /// The key is not supported + internal static void ValidateKey(string key) + { + var issues = GetKeyIssues(key); + if (issues != null) + { + throw new ArgumentException($"ObjectStore: path is not supported: '{key}': {issues}"); + } + } + + /// + /// The names the objects are stored under, then the unsupported keys saved in this run whose object exists + /// + private ICollection ListKeys() + { + var keys = _store.Keys; + if (_sanitizedKeys.IsEmpty) + { + return keys; + } + var unsupportedKeys = _sanitizedKeys.Where(pair => _store.ContainsKey(pair.Value)).Select(pair => pair.Key).ToList(); + return unsupportedKeys.Count == 0 ? keys : keys.Union(unsupportedKeys).ToList(); + } + + /// + /// The key a save goes to: the given one when it follows the key format, else its sanitized version, replacing + /// the object stored there. Null and empty keys are left for the store to reject + /// + private string KeyToSave(string path) + { + if (string.IsNullOrEmpty(path) || IsSupportedKey(path)) + { + return path; + } + + var key = SanitizeKey(path); + _sanitizedKeys[path] = key; + WarnOnce(path, key); + return key; + } + + /// + /// The key the other operations use: the given one when it follows the key format, else its sanitized version, + /// so an object is found by the key it was saved with. An object stored under the name as given, before the + /// key format applied, is used while the sanitized one does not exist + /// + private string KeyToRead(string path) + { + if (string.IsNullOrEmpty(path) || IsSupportedKey(path)) + { + return path; + } + + var key = SanitizeKey(path); + if (!_store.ContainsKey(key) && _store.ContainsKey(path)) + { + return path; + } + WarnOnce(path, key); + return key; + } + + private void WarnOnce(string path, string key) + { + if (Interlocked.Exchange(ref _warned, 1) == 0) + { + _warn($"ObjectStore: key '{path}' is not supported: {GetKeyIssues(path)}. Using '{key}' instead." + + " Any other unsupported key is sanitized the same way"); + } + } + + private static bool IsSupportedKey(string key) + { + return !string.IsNullOrEmpty(key) && !UnsupportedKeyCharactersRegex.IsMatch(key) && !key.Contains("..", StringComparison.Ordinal) && HasName(key) + // just in case + && key.Count(c => c == '/') <= 100 && key.Count(c => c == '\\') <= 100; + } + + /// + /// Whether the key has something besides dots and separators, so it names an object and not a folder + /// + private static bool HasName(string key) + { + return key.Any(c => c != '.' && c != '/' && c != '\\'); + } + + /// + /// Describes why a key does not follow the key format: the problems found and the rules. Null for a supported key + /// + private static string GetKeyIssues(string key) + { + if (IsSupportedKey(key)) + { + return null; + } + return string.Join("; ", FindKeyIssues(key)) + + ". Keys may only contain english letters, numbers, spaces and the characters" + + " '.', '/', '\\', '_', '#', '-', '$' and '=', never two '.' in a row, e.g. 'folder/trade_log-2024.v2.csv'"; + } + + /// + /// Lists why a key fails , one entry per problem found + /// + private static List FindKeyIssues(string key) + { + var issues = new List(); + if (string.IsNullOrEmpty(key)) + { + issues.Add("the key is empty"); + return issues; + } + + var unsupported = UnsupportedKeyCharactersRegex.Matches(key).Select(match => $"'{match.Value}'").Distinct().ToList(); + if (unsupported.Count > 0) + { + issues.Add($"unsupported character{(unsupported.Count > 1 ? "s" : "")} {string.Join(", ", unsupported)}"); + } + + if (key.Contains("..", StringComparison.Ordinal)) + { + issues.Add("it contains '..'"); + } + if (!HasName(key)) + { + issues.Add("it has no name, only '.' and separators"); + } + + if (key.Count(c => c == '/') > 100 || key.Count(c => c == '\\') > 100) + { + issues.Add("it has more than 100 '/' or '\\' separators"); + } + + if (issues.Count == 0) + { + issues.Add("it does not follow the key format"); + } + return issues; + } + + /// + /// Converts an unsupported key into a supported one by replacing every unsupported character, and every '.' + /// of a '..' run, with '_': 'trade log: AI & Cloud.csv' becomes 'trade log_ AI _ Cloud.csv'. + /// The mapping is lossy, distinct keys can sanitize to the same one. Throws when the key cannot be fixed + /// + private static string SanitizeKey(string key) + { + var sanitized = UnsupportedKeyCharactersRegex.Replace(key, "_"); + sanitized = ConsecutiveDotsRegex.Replace(sanitized, match => new string('_', match.Length)); + + ValidateKey(sanitized); + return sanitized; + } + + /// + /// The keys of the object store, listed on first use. Its Contains finds a key the way + /// does, without listing the store + /// + private sealed class KeyCollection : ICollection + { + private readonly Lazy> _keys; + private readonly Func _containsKey; + + public int Count => _keys.Value.Count; + + public bool IsReadOnly => _keys.Value.IsReadOnly; + + public KeyCollection(Func> listKeys, Func containsKey) + { + _keys = new Lazy>(listKeys); + _containsKey = containsKey; + } + + public bool Contains(string item) + { + return !string.IsNullOrEmpty(item) && _containsKey(item); + } + + /// + /// Python's 'in' operator calls this on a type that also has a Count + /// + public bool ContainsKey(string key) + { + return Contains(key); + } + + public void Add(string item) + { + _keys.Value.Add(item); + } + + public void Clear() + { + _keys.Value.Clear(); + } + + public bool Remove(string item) + { + return _keys.Value.Remove(item); + } + + public void CopyTo(string[] array, int arrayIndex) + { + _keys.Value.CopyTo(array, arrayIndex); + } + + public IEnumerator GetEnumerator() + { + return _keys.Value.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + } } } diff --git a/Common/Storage/PythonObjectStore.cs b/Common/Storage/PythonObjectStore.cs index a6792db05535..a0d64ec199bf 100644 --- a/Common/Storage/PythonObjectStore.cs +++ b/Common/Storage/PythonObjectStore.cs @@ -13,6 +13,7 @@ * limitations under the License. */ +using System; using Python.Runtime; using QuantConnect.Interfaces; @@ -34,8 +35,9 @@ public class PythonObjectStore : ObjectStore /// Initializes a new instance of the class /// /// The instance to wrap - public PythonObjectStore(IObjectStore store) - : base(store) + /// Receives the warnings about sanitized keys, the engine log by default + public PythonObjectStore(IObjectStore store, Action warn = null) + : base(store, warn) { } @@ -85,7 +87,7 @@ public PyObject ReadDataframe(string path) /// , and so is any other type that would not round trip. A top-level Python /// str, int, float or bool is not accepted, use for text /// - /// The object path + /// The object key: english letters, numbers, spaces and the characters '.', '/', '\', '_', '#', '-', '$' and '=', never '..', e.g. 'folder/trade_log-2024.v2.csv'. Other characters are replaced by '_' /// The Python object to be saved /// True if the object was saved successfully /// A TypeError naming the value that cannot be serialized @@ -105,7 +107,7 @@ public bool SaveJson(string path, PyObject obj) /// Saves the given pandas DataFrame (or Series) for the specified path, in JSON format if the /// path has a '.json' extension and as CSV otherwise /// - /// The object path + /// The object key: english letters, numbers, spaces and the characters '.', '/', '\', '_', '#', '-', '$' and '=', never '..', e.g. 'folder/trade_log-2024.v2.csv'. Other characters are replaced by '_' /// The pandas DataFrame or Series to be saved /// True if the object was saved successfully public bool SaveDataframe(string path, PyObject dataFrame) diff --git a/Engine/Storage/LocalObjectStore.cs b/Engine/Storage/LocalObjectStore.cs index bbe6d7ccaf81..679ef52f414c 100644 --- a/Engine/Storage/LocalObjectStore.cs +++ b/Engine/Storage/LocalObjectStore.cs @@ -19,7 +19,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text.RegularExpressions; using System.Threading; using QuantConnect.Configuration; using QuantConnect.Interfaces; @@ -112,7 +111,6 @@ private bool IsDirty } private Timer _persistenceTimer; - private Regex _pathRegex = new(@"^\.?[a-zA-Z0-9\\/_#\-\$= ]+\.?[a-zA-Z0-9]*$", RegexOptions.Compiled); private readonly ConcurrentDictionary _storage = new(); private readonly object _persistLock = new object(); @@ -333,15 +331,7 @@ public bool SaveBytes(string path, byte[] contents) { throw new InvalidOperationException($"LocalObjectStore.SaveBytes(): {NoWritePermissionsError}"); } - else if (!_pathRegex.IsMatch(path)) - { - throw new ArgumentException($"LocalObjectStore: path is not supported: '{path}'"); - } - else if (path.Count(c => c == '/') > 100 || path.Count(c => c == '\\') > 100) - { - // just in case - throw new ArgumentException($"LocalObjectStore: path is not supported: '{path}'"); - } + ObjectStore.ValidateKey(path); // after we check the regex path = NormalizePath(path); diff --git a/Research/QuantBook.cs b/Research/QuantBook.cs index f53cab5681b8..ad9678d43726 100644 --- a/Research/QuantBook.cs +++ b/Research/QuantBook.cs @@ -166,6 +166,7 @@ public QuantBook() : base() StorageAccess = Config.GetValue("storage-permissions", new Packets.StoragePermissions()) }, AlgorithmMode.Research); + // key warnings go to the engine log, like every other QuantBook message: nothing drains the debug messages here SetObjectStore(_isPythonNotebook ? new PythonObjectStore(algorithmHandlers.ObjectStore) : new ObjectStore(algorithmHandlers.ObjectStore)); diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index 4f6b6e42892a..ca883f9e4700 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -27,6 +27,7 @@ using System.Threading.Tasks; using System.Collections.Generic; using QuantConnect.Lean.Engine.Storage; +using QuantConnect.Lean.Engine.DataFeeds.Transport; using System.Reflection; using Python.Runtime; @@ -234,9 +235,9 @@ public void DeletePermissions([Values] bool read, [Values] bool write, [Values] } } - [TestCase("../prefix/")] - [TestCase("..\\prefix/")] - public void InvalidCustomPathsStore(string path) + [TestCase("../prefix/", "__/prefix/")] + [TestCase("..\\prefix/", "__/prefix/")] + public void InvalidCustomPathsStore(string path, string sanitizedPath) { #pragma warning disable CA2000 using (var store = new ObjectStore(new TestLocalObjectStore())) @@ -245,7 +246,12 @@ public void InvalidCustomPathsStore(string path) store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); Assert.AreEqual(0, store.Count()); - Assert.Throws(() => store.SaveString($"{path}ILove", "Pizza")); + // '..' cannot leave the storage root: it is sanitized away + Assert.IsTrue(store.SaveString($"{path}ILove", "Pizza")); + Assert.AreEqual(new[] { $"{sanitizedPath}ILove", $"{path}ILove" }, store.Keys); + Assert.That(Path.GetFullPath(store.GetFilePath($"{path}ILove")), Does.StartWith(Path.GetFullPath(TestStorageRoot))); + + store.Delete($"{path}ILove"); } } @@ -622,22 +628,496 @@ public void SaveTextAndReadTextAreAliasesOfSaveAndRead() _store.Delete("save_text_alias.txt"); } - [TestCase("my_key", "./LocalObjectStoreTests/my_key")] - [TestCase("test/123", "./LocalObjectStoreTests/test/123")] - [TestCase("**abc**", null)] - [TestCase("", null)] - [TestCase("|", null)] - public void GetFilePathReturnsFileName(string key, string expectedRelativePath) + [TestCase("my_key/file.csv", "my_key/file.csv")] + [TestCase("folder.v1/file.tar.gz", "folder.v1/file.tar.gz")] + [TestCase("my.folder/file.name_v2", "my.folder/file.name_v2")] + [TestCase("trade_log_ai_hardware_&_cloud.csv", "trade_log_ai_hardware___cloud.csv")] + [TestCase("trade log: AI & Cloud.csv", "trade log_ AI _ Cloud.csv")] + [TestCase("résumé.pdf", "r_sum_.pdf")] + [TestCase("a..", "a__")] + [TestCase("100%.json", "100_.json")] + [TestCase("..model.pkl", "__model.pkl")] + [TestCase("a/../b.csv", "a/__/b.csv")] + [TestCase("a...b.csv", "a___b.csv")] + [TestCase("file.csv\n", "file.csv_")] + public void UnsupportedKeysAreSanitizedOnSaveAndFoundByTheKeyTheyWereSavedWith(string key, string sanitizedKey) + { + Assert.IsTrue(_store.Save(key, "abc")); + + Assert.IsTrue(_store.Keys.Contains(sanitizedKey), string.Join(", ", _store.Keys)); + Assert.That(_store.Keys.ToList(), Does.Contain(key)); + Assert.IsTrue(_store.ContainsKey(key)); + Assert.IsTrue(_store.ContainsKey(sanitizedKey)); + Assert.AreEqual("abc", _store.Read(key)); + Assert.AreEqual("abc", _store.Read(sanitizedKey)); + + Assert.IsTrue(_store.Delete(key)); + Assert.IsFalse(_store.ContainsKey(sanitizedKey)); + Assert.That(_store.Keys.ToList(), Does.Not.Contain(key)); + } + + [TestCase("../secrets.json", "__/secrets.json")] + [TestCase("../../secrets.json", "__/__/secrets.json")] + [TestCase("..\\secrets.json", "__\\secrets.json")] + [TestCase("x/../secrets.json", "x/__/secrets.json")] + [TestCase("x/../../secrets.json", "x/__/__/secrets.json")] + [TestCase("x\\..\\..\\secrets.json", "x\\__\\__\\secrets.json")] + [TestCase("x/.../secrets.json", "x/___/secrets.json")] + [TestCase("x/..", "x/__")] + [TestCase("x/....", "x/____")] + public void TraversalKeysAreSanitizedAndStayInsideTheStorageRoot(string key, string sanitizedKey) + { + // '..' never reaches the file system, so a key cannot point at another folder's or organization's objects + var warnings = new List(); + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + var root = Path.GetFullPath(TestStorageRoot) + Path.DirectorySeparatorChar; + + Assert.That(Path.GetFullPath(store.GetFilePath(key)), Does.StartWith(root)); + Assert.IsTrue(store.Save(key, "abc")); + + Assert.IsTrue(store.Keys.Contains(sanitizedKey.Replace('\\', '/')), string.Join(", ", store.Keys)); + // the keys list the key as saved too, the objects are only stored under names without '..' + Assert.That(store.Keys.ToList(), Does.Contain(key)); + Assert.IsFalse(store.Any(entry => entry.Key.Contains("..")), string.Join(", ", store.Select(entry => entry.Key))); + Assert.That(Path.GetFullPath(store.GetFilePath(sanitizedKey)), Does.StartWith(root)); + Assert.AreEqual("abc", store.Read(key)); + Assert.That(warnings[0], Does.Contain("it contains '..'")); + + Assert.IsTrue(store.Delete(key)); + Assert.IsFalse(store.ContainsKey(sanitizedKey)); + } + + [Test] + public void GetFilePathOfAnUnsupportedKeyIsTheFileTheSaveThatFollowsReads() { - if (expectedRelativePath == null) + var warnings = new List(); + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + + var filePath = store.GetFilePath("model (1).pkl"); + Assert.AreEqual(Path.GetFullPath(Path.Combine(TestStorageRoot, "model _1_.pkl")), Path.GetFullPath(filePath)); + File.WriteAllText(filePath, "abc"); + + Assert.IsTrue(store.Save("model (1).pkl")); + + Assert.AreEqual("abc", store.Read("model (1).pkl")); + Assert.AreEqual("abc", store.Read("model _1_.pkl")); + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); + Assert.IsTrue(store.Delete("model (1).pkl")); + Assert.IsFalse(store.ContainsKey("model _1_.pkl")); + } + + [Test] + public void UnsupportedKeysAreWarnedAboutOnceStatingTheIssueAndTheKeyUsed() + { + var warnings = new List(); + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + + Assert.IsTrue(store.Save("trade log: AI & Cloud.csv", "abc")); + Assert.IsTrue(store.ContainsKey("trade log: AI & Cloud.csv")); + Assert.AreEqual("abc", store.Read("trade log: AI & Cloud.csv")); + Assert.IsTrue(store.Save("trade log: AI & Cloud.csv", "abcd")); + Assert.IsTrue(store.Save("supported.csv", "abc")); + + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); + Assert.AreEqual("ObjectStore: key 'trade log: AI & Cloud.csv' is not supported: unsupported characters ':', '&'. " + + "Keys may only contain english letters, numbers, spaces and the characters '.', '/', '\\', '_', '#', '-', '$' and '=', " + + "never two '.' in a row, e.g. 'folder/trade_log-2024.v2.csv'. " + + "Using 'trade log_ AI _ Cloud.csv' instead. Any other unsupported key is sanitized the same way", warnings[0]); + + // the next unsupported key is sanitized silently + Assert.IsFalse(store.ContainsKey("trade log: AI.v2.csv")); + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); + + store.Delete("trade log: AI & Cloud.csv"); + store.Delete("supported.csv"); + } + + [TestCase("trade log: AI & Cloud.csv", "unsupported characters ':', '&'")] + [TestCase("résumé.pdf", "unsupported character 'é'")] + [TestCase("..model.pkl", "it contains '..'")] + [TestCase("a/../b.csv", "it contains '..'")] + [TestCase("trade log: AI..v2.csv", "unsupported character ':'; it contains '..'")] + public void UnsupportedKeyWarningStatesTheIssue(string key, string expectedIssue) + { + var warnings = new List(); + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + + Assert.IsFalse(store.ContainsKey(key)); + + Assert.AreEqual(1, warnings.Count); + Assert.That(warnings[0], Does.StartWith($"ObjectStore: key '{key}' is not supported: {expectedIssue}.")); + } + + [TestCase("", "the key is empty")] + [TestCase(null, null)] + [TestCase(".", "it has no name, only '.' and separators")] + [TestCase("./", "it has no name, only '.' and separators")] + [TestCase("/", "it has no name, only '.' and separators")] + public void EmptyAndNamelessKeysAreRejected(string key, string expectedIssue) + { + // nothing to sanitize these into: they throw like the store itself would, without a warning + var warnings = new List(); + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + + var error = Assert.Catch(() => store.Save(key, "abc")); + + Assert.IsEmpty(warnings); + if (expectedIssue != null) { - Assert.Throws(() => _store.GetFilePath(key)); + Assert.That(error.Message, Does.Contain($"path is not supported: '{key}': {expectedIssue}.")); } - else + } + + [TestCase("model (1).tar.gz")] + [TestCase("100%.json")] + [TestCase("BRK&B.csv")] + public void ExistingUnsupportedKeysAreUsedAsGiven(string key) + { + // an object put in the store by other means (uploaded, copied into the folder) under an unsupported name + var warnings = new List(); + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + var filePath = Path.Combine(TestStorageRoot, key); + File.WriteAllText(filePath, "abc"); + + Assert.IsTrue(store.ContainsKey(key)); + Assert.AreEqual("abc", store.Read(key)); + Assert.IsTrue(store.Keys.Contains(key), string.Join(", ", store.Keys)); + Assert.AreEqual(Path.GetFullPath(filePath), Path.GetFullPath(store.GetFilePath(key))); + Assert.IsEmpty(warnings, string.Join("\n", warnings)); + Assert.IsTrue(store.Delete(key)); + Assert.IsFalse(File.Exists(filePath)); + + // gone from the store, the key falls back to its sanitized name like any other unsupported key + Assert.IsFalse(store.ContainsKey(key)); + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); + Assert.That(warnings[0], Does.StartWith($"ObjectStore: key '{key}' is not supported:")); + } + + [Test] + public void SavingToAnExistingUnsupportedKeyGoesToTheSanitizedNameWhichReadsThenReturn() + { + // an object put in the store by other means under an unsupported name keeps it: saves through that name go to the sanitized one + var filePath = Path.Combine(TestStorageRoot, "model (1).tar.gz"); + File.WriteAllText(filePath, "old"); + var warnings = new List(); + using (var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add)) { - var expectedPath = Path.GetFullPath(expectedRelativePath).Replace("\\", "/"); - Assert.AreEqual(expectedPath, _store.GetFilePath(key).Replace("\\", "/")); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.AreEqual("old", store.Read("model (1).tar.gz")); + Assert.IsEmpty(warnings, string.Join("\n", warnings)); + + Assert.IsTrue(store.Save("model (1).tar.gz", "new")); + + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); + Assert.That(warnings[0], Does.StartWith("ObjectStore: key 'model (1).tar.gz' is not supported: unsupported characters '(', ')'.")); + Assert.AreEqual("new", store.Read("model (1).tar.gz")); + Assert.AreEqual("new", store.Read("model _1_.tar.gz")); + Assert.AreEqual("old", File.ReadAllText(filePath)); } + + // a later run reads and replaces the sanitized object through the same name + using (var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add)) + { + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.AreEqual("new", store.Read("model (1).tar.gz")); + Assert.IsTrue(store.Save("model (1).tar.gz", "newer")); + Assert.AreEqual("newer", store.Read("model _1_.tar.gz")); + + // with the sanitized object gone, the one stored under the name as given is read again + Assert.IsTrue(store.Delete("model (1).tar.gz")); + Assert.IsFalse(store.ContainsKey("model _1_.tar.gz")); + Assert.AreEqual("old", store.Read("model (1).tar.gz")); + Assert.IsTrue(store.Delete("model (1).tar.gz")); + Assert.IsFalse(store.ContainsKey("model (1).tar.gz")); + } + } + + [Test] + public void DeletingASavedUnsupportedKeyRemovesTheSavedObjectThenTheOriginal() + { + var filePath = Path.Combine(TestStorageRoot, "model (1).tar.gz"); + File.WriteAllText(filePath, "old"); + using var store = new ObjectStore(new TestLocalObjectStore(), _ => { }); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.IsTrue(store.Save("model (1).tar.gz", "new")); + Assert.AreEqual("new", store.Read("model (1).tar.gz")); + + // the first delete removes what this store saved, and the key resolves like a fresh store's from then on + Assert.IsTrue(store.Delete("model (1).tar.gz")); + Assert.IsFalse(File.Exists(Path.Combine(TestStorageRoot, "model _1_.tar.gz"))); + Assert.IsTrue(store.ContainsKey("model (1).tar.gz")); + Assert.AreEqual("old", store.Read("model (1).tar.gz")); + + Assert.IsTrue(store.Delete("model (1).tar.gz")); + Assert.IsFalse(store.ContainsKey("model (1).tar.gz")); + Assert.IsFalse(File.Exists(filePath)); + } + + [Test] + public void AFailedSaveToAnUnsupportedKeyDoesNotRedirectItsReads() + { + var filePath = Path.Combine(TestStorageRoot, "model (1).tar.gz"); + File.WriteAllText(filePath, "old"); + using var store = new ObjectStore(new TestLocalObjectStore(), _ => { }); + // the 3 bytes stored fill the storage: the save is rejected, not thrown, in backtesting + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1, StorageLimit = 3 }, AlgorithmMode.Backtesting); + + Assert.IsFalse(store.Save("model (1).tar.gz", "new")); + + Assert.IsTrue(store.ContainsKey("model (1).tar.gz")); + Assert.AreEqual("old", store.Read("model (1).tar.gz")); + Assert.IsFalse(store.ContainsKey("model _1_.tar.gz")); + + Assert.IsTrue(store.Delete("model (1).tar.gz")); + Assert.IsFalse(File.Exists(filePath)); + } + + [Test] + public void CustomDataIsReadFromAnExistingUnsupportedKey() + { + using var store = new ObjectStore(new TestLocalObjectStore()); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + File.WriteAllText(Path.Combine(TestStorageRoot, "BRK&B.csv"), "2013-10-07,167.42\n2013-10-08,165.48\n"); + + // the data feed reads custom data through the algorithm's store + using var reader = new ObjectStoreSubscriptionStreamReader(store, "BRK&B.csv"); + + Assert.IsFalse(reader.EndOfStream); + Assert.AreEqual("2013-10-07,167.42", reader.ReadLine()); + Assert.AreEqual("2013-10-08,165.48", reader.ReadLine()); + store.Delete("BRK&B.csv"); + } + + [Test] + public void SavingAnUnsupportedKeyReplacesTheObjectUnderItsSanitizedName() + { + var warnings = new List(); + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.IsTrue(store.Save("p_q.csv", "direct")); + + Assert.IsTrue(store.Save("p&q.csv", "ampersand")); + Assert.AreEqual("ampersand", store.Read("p_q.csv")); + + // distinct unsupported keys can share a sanitized name: the last save wins + Assert.IsTrue(store.Save("p:q.csv", "colon")); + Assert.AreEqual("colon", store.Read("p&q.csv")); + Assert.AreEqual("colon", store.Read("p_q.csv")); + Assert.AreEqual(1, warnings.Count, string.Join("\n", warnings)); + + Assert.IsTrue(store.Delete("p&q.csv")); + Assert.IsFalse(store.ContainsKey("p_q.csv")); + } + + [Test] + public void ANewRunSavesThroughTheSameUnsupportedKeyAgain() + { + for (var run = 1; run <= 2; run++) + { + using var store = new ObjectStore(new TestLocalObjectStore(), _ => { }); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + + Assert.IsTrue(store.Save("trade log: AI & Cloud.csv", $"run{run}")); + + Assert.AreEqual($"run{run}", store.Read("trade log: AI & Cloud.csv")); + Assert.AreEqual($"run{run}", store.Read("trade log_ AI _ Cloud.csv")); + } + + using var lastRun = new ObjectStore(new TestLocalObjectStore(), _ => { }); + lastRun.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.IsTrue(lastRun.Delete("trade log: AI & Cloud.csv")); + } + + [Test] + public void ABackslashKeyIsSavedInEveryRunAsOnTheStoreItAlwaysReadAsASeparator() + { + var warnings = new List(); + for (var run = 1; run <= 2; run++) + { + using var store = new ObjectStore(new TestLocalObjectStore(), warnings.Add); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + + Assert.IsTrue(store.Save("backslash_folder\\results.csv", $"run{run}")); + + Assert.AreEqual($"run{run}", store.Read("backslash_folder\\results.csv")); + Assert.AreEqual($"run{run}", store.Read("backslash_folder/results.csv")); + Assert.IsTrue(store.Keys.Contains("backslash_folder/results.csv"), string.Join(", ", store.Keys)); + Assert.IsTrue(store.ContainsKey("backslash_folder\\results.csv")); + } + Assert.IsEmpty(warnings, string.Join("\n", warnings)); + + using var lastRun = new ObjectStore(new TestLocalObjectStore()); + lastRun.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.IsTrue(lastRun.Delete("backslash_folder\\results.csv")); + Assert.IsFalse(lastRun.ContainsKey("backslash_folder/results.csv")); + } + + [Test] + public void KeysHoldAnUnsupportedKeySavedInThisRunAndItsSanitizedName() + { + using var store = new ObjectStore(new TestLocalObjectStore(), _ => { }); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.IsTrue(store.Save("keys check: A & B.csv", "abc")); + Assert.IsTrue(store.Save("keys check: A & B.csv", "def")); + Assert.IsTrue(store.Save("keys_folder\\file.csv", "abc")); + + var keys = store.Keys; + + var listed = keys.ToList(); + Assert.AreEqual(1, listed.Count(key => key == "keys check: A & B.csv"), string.Join(", ", listed)); + Assert.AreEqual(1, listed.Count(key => key == "keys check_ A _ B.csv"), string.Join(", ", listed)); + Assert.That(listed, Does.Contain("keys_folder/file.csv")); + Assert.AreEqual(listed.Count, keys.Count); + var copy = new string[keys.Count]; + keys.CopyTo(copy, 0); + Assert.AreEqual(listed, copy); + + Assert.IsTrue(keys.Contains("keys check: A & B.csv")); + Assert.IsTrue(keys.Contains("keys check_ A _ B.csv")); + Assert.IsTrue(keys.Contains("keys_folder\\file.csv")); + Assert.IsTrue(keys.Contains("keys_folder/file.csv")); + Assert.IsFalse(keys.Contains("keys_missing.csv")); + Assert.IsFalse(keys.Contains(null)); + + // a run that did not save through the unsupported key lists the stored name only, and still finds the key + using (var otherRun = new ObjectStore(new TestLocalObjectStore(), _ => { })) + { + otherRun.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.That(otherRun.Keys.ToList(), Does.Contain("keys check_ A _ B.csv")); + Assert.That(otherRun.Keys.ToList(), Does.Not.Contain("keys check: A & B.csv")); + Assert.IsTrue(otherRun.Keys.Contains("keys check: A & B.csv")); + } + + // the unsupported key is listed while the object it was saved to exists + Assert.IsTrue(store.Delete("keys check_ A _ B.csv")); + Assert.That(store.Keys.ToList(), Does.Not.Contain("keys check: A & B.csv")); + Assert.IsFalse(store.Keys.Contains("keys check: A & B.csv")); + Assert.IsTrue(store.Delete("keys_folder\\file.csv")); + } + + [Test] + public void KeysListAnExistingUnsupportedNameOnceAfterASaveThroughIt() + { + using var store = new ObjectStore(new TestLocalObjectStore(), _ => { }); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + File.WriteAllText(Path.Combine(TestStorageRoot, "keys legacy (1).csv"), "old"); + + Assert.IsTrue(store.Save("keys legacy (1).csv", "new")); + + var listed = store.Keys.ToList(); + Assert.AreEqual(1, listed.Count(key => key == "keys legacy (1).csv"), string.Join(", ", listed)); + Assert.AreEqual(1, listed.Count(key => key == "keys legacy _1_.csv"), string.Join(", ", listed)); + + Assert.IsTrue(store.Delete("keys legacy _1_.csv")); + Assert.IsTrue(store.Delete("keys legacy (1).csv")); + Assert.That(store.Keys.ToList(), Does.Not.Contain("keys legacy (1).csv")); + } + + [Test] + public void KeysListTheStoreOnFirstUseOnlyAndContainsNeverDoes() + { + var fileHandler = new ListingCountFileHandler(); + var localStore = new TestLocalObjectStore(); + using var store = new ObjectStore(localStore, _ => { }); + localStore.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, fileHandler); + Assert.IsTrue(store.Save("lazy keys: A & B.csv", "abc")); + + var listings = fileHandler.Listings; + var keys = store.Keys; + Assert.IsTrue(keys.Contains("lazy keys: A & B.csv")); + Assert.IsTrue(keys.Contains("lazy keys_ A _ B.csv")); + Assert.IsFalse(keys.Contains("lazy keys missing.csv")); + using (Py.GIL()) + { + var module = PyModule.FromString("TestLazyObjectStoreKeys", "def contains(keys, key):\n return key in keys\n"); + dynamic contains = module.GetAttr("contains"); + PyObject found = contains(store.Keys, "lazy keys: A & B.csv"); + PyObject missing = contains(store.Keys, "lazy keys missing.csv"); + Assert.IsTrue(found.As()); + Assert.IsFalse(missing.As()); + } + Assert.AreEqual(listings, fileHandler.Listings); + + // the listing is built on first use, once per collection + var listed = keys.ToList(); + Assert.AreEqual(listed.Count, keys.Count); + Assert.AreEqual(listed, keys.ToList()); + Assert.AreEqual(listings + 1, fileHandler.Listings); + Assert.That(listed, Does.Contain("lazy keys: A & B.csv")); + Assert.That(listed, Does.Contain("lazy keys_ A _ B.csv")); + + // a new access lists again, so it sees what changed since + Assert.IsTrue(store.Save("lazy_keys_later.csv", "abc")); + listings = fileHandler.Listings; + Assert.That(store.Keys.ToList(), Does.Contain("lazy_keys_later.csv")); + Assert.AreEqual(listings + 1, fileHandler.Listings); + + Assert.IsTrue(store.Delete("lazy keys: A & B.csv")); + Assert.IsTrue(store.Delete("lazy_keys_later.csv")); + } + + [Test] + public void QuantBookWarnsThroughTheEngineLog() + { + var traces = new List(); + Log.LogHandler = new FunctionalLogHandler(null, traces.Add, null); + var quantBook = new QuantBook(); + Assert.AreEqual(typeof(ObjectStore), quantBook.ObjectStore.GetType()); + + Assert.IsTrue(quantBook.ObjectStore.Save("trade log: AI & Cloud.csv", "abc")); + + Assert.AreEqual(1, traces.Count(trace => trace.Contains("ObjectStore: key 'trade log: AI & Cloud.csv' is not supported: unsupported characters ':', '&'.")), + string.Join("\n", traces)); + quantBook.ObjectStore.Delete("trade log: AI & Cloud.csv"); + } + + [Test] + public void LocalObjectStoreRejectsUnsupportedKeysStatingTheIssue() + { + // the implementation is the last line of defense, only the wrapper sanitizes + using var store = new TestLocalObjectStore(); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }); + + var error = Assert.Throws(() => store.SaveBytes("trade_log_ai_hardware_&_cloud.csv", new byte[] { 1 })); + + Assert.AreEqual("ObjectStore: path is not supported: 'trade_log_ai_hardware_&_cloud.csv': unsupported character '&'. " + + "Keys may only contain english letters, numbers, spaces and the characters '.', '/', '\\', '_', '#', '-', '$' and '=', " + + "never two '.' in a row, e.g. 'folder/trade_log-2024.v2.csv'", error.Message); + } + + [Test] + public void AlgorithmWrapsTheStoreAndWarnsThroughDebug() + { + using var localStore = new TestLocalObjectStore(); + localStore.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }); + var algorithm = new QCAlgorithm(); + + algorithm.SetObjectStore(localStore); + + Assert.AreEqual(typeof(ObjectStore), algorithm.ObjectStore.GetType()); + Assert.IsTrue(algorithm.ObjectStore.Save("trade_log_ai_hardware_&_cloud.csv", "abc")); + Assert.IsTrue(algorithm.DebugMessages.TryDequeue(out var warning)); + Assert.That(warning, Does.Contain("ObjectStore: key 'trade_log_ai_hardware_&_cloud.csv' is not supported: unsupported character '&'.")); + Assert.That(warning, Does.Contain("Using 'trade_log_ai_hardware___cloud.csv' instead")); + + algorithm.ObjectStore.Delete("trade_log_ai_hardware_&_cloud.csv"); + } + + [TestCase("my_key", "./LocalObjectStoreTests/my_key")] + [TestCase("test/123", "./LocalObjectStoreTests/test/123")] + [TestCase("**abc**", "./LocalObjectStoreTests/__abc__")] + [TestCase("", "./LocalObjectStoreTests/_random_")] + [TestCase("|", "./LocalObjectStoreTests/_")] + public void GetFilePathReturnsFileName(string key, string expectedRelativePath) + { + var expectedPath = Path.GetFullPath(expectedRelativePath).Replace("\\", "/"); + Assert.AreEqual(expectedPath, _store.GetFilePath(key).Replace("\\", "/")); } [Test] @@ -1413,6 +1893,41 @@ def test(object_store): "); } + [Test] + public void PythonHelpersSanitizeUnsupportedKeys() + { + RunPythonTest(@" +def test(object_store): + object_store.save_json('helpers/trade log: AI & Cloud.json', {'a': 1}) + assert 'helpers/trade log_ AI _ Cloud.json' in list(object_store.keys), list(object_store.keys) + assert object_store.read_json('helpers/trade log: AI & Cloud.json') == {'a': 1} + object_store.delete('helpers/trade log: AI & Cloud.json') + assert not object_store.contains_key('helpers/trade log_ AI _ Cloud.json') +"); + } + + [Test] + public void PythonKeysHoldAnUnsupportedKeySavedInThisRunAndItsSanitizedName() + { + RunPythonTest(@" +def test(object_store): + key = 'helpers/in check: A & B.csv' + object_store.save(key, 'abc') + + assert key in object_store.keys + assert 'helpers/in check_ A _ B.csv' in object_store.keys + assert 'helpers/missing.csv' not in object_store.keys + # the keys list the key as saved next to the name the object is stored under + assert list(object_store.keys).count('helpers/in check_ A _ B.csv') == 1 + assert list(object_store.keys).count(key) == 1 + assert len(object_store.keys) == len(list(object_store.keys)) + + object_store.delete(key) + assert key not in object_store.keys + assert key not in list(object_store.keys) +"); + } + /// /// Runs the 'test(object_store)' function of the given Python source against the test store /// @@ -1483,6 +1998,17 @@ protected override bool PersistData() protected override string StorageRoot() => TestStorageRoot; } + private sealed class ListingCountFileHandler : FileHandler + { + public int Listings { get; private set; } + + public override IEnumerable EnumerateFiles(string path, string pattern, SearchOption searchOption, out string rootfolder) + { + Listings++; + return base.EnumerateFiles(path, pattern, searchOption, out rootfolder); + } + } + public class TestFileHandler : FileHandler { public override void WriteAllBytes(string path, byte[] data) diff --git a/Tests/Python/AlgorithmPythonWrapperTests.cs b/Tests/Python/AlgorithmPythonWrapperTests.cs index 2dcab23c97a5..703b6206b284 100644 --- a/Tests/Python/AlgorithmPythonWrapperTests.cs +++ b/Tests/Python/AlgorithmPythonWrapperTests.cs @@ -173,6 +173,29 @@ public void PythonAlgorithmGetsAPythonObjectStore() } } + [Test] + public void PythonAlgorithmKeyWarningGoesThroughDebug() + { + using (Py.GIL()) + { + var algorithm = GetAlgorithm("def on_end_of_algorithm(self): self.object_store.save('trade log: AI & Cloud.json', '{}')"); + using var localStore = new LocalObjectStore(); + localStore.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + + algorithm.SetObjectStore(localStore); + + // the key warning lands in the algorithm's log + algorithm.OnEndOfAlgorithm(); + Assert.Null(algorithm.RunTimeError); + Assert.IsTrue(localStore.Keys.Contains("trade log_ AI _ Cloud.json"), string.Join(", ", localStore.Keys)); + Assert.IsTrue(algorithm.BaseAlgorithm.DebugMessages.TryDequeue(out var warning)); + Assert.That(warning, Does.Contain("ObjectStore: key 'trade log: AI & Cloud.json' is not supported: unsupported characters ':', '&'.")); + Assert.That(warning, Does.Contain("Using 'trade log_ AI _ Cloud.json' instead")); + + localStore.Delete("trade log_ AI _ Cloud.json"); + } + } + private AlgorithmPythonWrapper GetAlgorithm(string code) { code = $"{_baseCode}{Environment.NewLine} {code}";