diff --git a/QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs b/QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs index 1bb6604..b6ca90d 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs @@ -17,7 +17,9 @@ using System; using System.Linq; using NUnit.Framework; +using QuantConnect.Securities; using System.Collections.Generic; +using QuantConnect.Configuration; namespace QuantConnect.Lean.DataSource.IQFeed.Tests { @@ -26,9 +28,16 @@ public class IQFeedDataDownloaderTest { private IQFeedDataDownloader _downloader; - [SetUp] - public void SetUp() + [OneTimeSetUp] + public void OneTimeSetUp() { + if (OS.IsWindows) + { + // IQConnect is only supported on Windows + var connector = new IQConnect(Config.Get("iqfeed-productName"), "1.0"); + Assert.IsTrue(connector.Launch(), "Failed to launch IQConnect on Windows. Ensure IQFeed is installed and configured properly."); + } + _downloader = new IQFeedDataDownloader(); } @@ -50,5 +59,35 @@ public void DownloadsHistoricalData(Symbol symbol, Resolution resolution, TickTy IQFeedHistoryProviderTests.AssertHistoricalDataResponse(resolution, downloadResponse); } + + private static IEnumerable CanonicalFutureSymbolTestCases + { + get + { + var startDateUtc = new DateTime(2025, 03, 03); + var endDateUtc = new DateTime(2025, 04, 04); + + var naturalGas = Symbol.Create(Futures.Energy.NaturalGas, SecurityType.Future, Market.NYMEX); + yield return new TestCaseData(naturalGas, Resolution.Daily, TickType.Trade, startDateUtc, endDateUtc); + + var nasdaq100EMini = Symbol.Create(Futures.Indices.NASDAQ100EMini, SecurityType.Future, Market.CME); + yield return new TestCaseData(nasdaq100EMini, Resolution.Daily, TickType.Trade, startDateUtc, endDateUtc); + } + } + + + [TestCaseSource(nameof(CanonicalFutureSymbolTestCases))] + public void DownloadCanonicalFutureHistoricalData(Symbol symbol, Resolution resolution, TickType tickType, DateTime startDateUtc, DateTime endDateUtc) + { + var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateUtc, endDateUtc, tickType); + var downloadResponse = _downloader.Get(parameters)?.ToList(); + + Assert.IsNotNull(downloadResponse); + Assert.IsNotEmpty(downloadResponse); + + var uniqueFutureSymbols = downloadResponse.Select(x => x.Symbol).Distinct().ToList(); + + Assert.That(uniqueFutureSymbols.Count, Is.GreaterThan(1), $"Expected more than 1 unique future symbol, but got {uniqueFutureSymbols.Count}: {string.Join(", ", uniqueFutureSymbols)}"); + } } } diff --git a/QuantConnect.IQFeed.Tests/IQFeedDataProviderTests.cs b/QuantConnect.IQFeed.Tests/IQFeedDataProviderTests.cs index 03e7728..67babf6 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedDataProviderTests.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedDataProviderTests.cs @@ -22,6 +22,7 @@ using QuantConnect.Tests; using QuantConnect.Logging; using System.Threading.Tasks; +using QuantConnect.Securities; using QuantConnect.Data.Market; using System.Collections.Generic; using QuantConnect.Lean.Engine.DataFeeds.Enumerators; @@ -52,6 +53,11 @@ private static IEnumerable SubscribeTestCaseData yield return new TestCaseData(Symbols.AAPL); yield return new TestCaseData(Symbol.Create("SMCI", SecurityType.Equity, Market.USA)); yield return new TestCaseData(Symbol.Create("IRBT", SecurityType.Equity, Market.USA)); + var nasdaq100EMini = Symbol.CreateFuture(Futures.Indices.NASDAQ100EMini, Market.CME, new DateTime(2025, 09, 19)); + yield return new TestCaseData(nasdaq100EMini); + var naturalGasAug2025 = Symbol.CreateFuture(Futures.Energy.NaturalGas, Market.NYMEX, new DateTime(2025, 08, 27)); + yield return new TestCaseData(naturalGasAug2025); + // yield return new TestCaseData(Symbols.SPY); // Not supported. @@ -127,19 +133,22 @@ private void SubscribeOnData(Symbol symbol, Resolution resolution, int minimumRe Action callback = (dataPoint) => { - if (dataPoint == null) + if (dataPoint is null) { return; } - switch (dataPoint) + if (dataPoint is Tick tick) { - case TradeBar _: - secondDataReceived[typeof(TradeBar)] += 1; - break; - case QuoteBar _: - secondDataReceived[typeof(QuoteBar)] += 1; - break; + switch (tick.TickType) + { + case TickType.Trade: + secondDataReceived[typeof(TradeBar)] += 1; + break; + case TickType.Quote: + secondDataReceived[typeof(QuoteBar)] += 1; + break; + } } if (secondDataReceived.All(type => type.Value >= minimumReturnDataAmount)) diff --git a/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs b/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs new file mode 100644 index 0000000..e1f9510 --- /dev/null +++ b/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs @@ -0,0 +1,57 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using System.Linq; +using NUnit.Framework; +using QuantConnect.Tests; +using QuantConnect.Interfaces; +using QuantConnect.Securities; +using System.Collections.Generic; + +namespace QuantConnect.Lean.DataSource.IQFeed.Tests; + +[TestFixture] +public class IQFeedDataQueueUniverseProviderTests +{ + private IDataQueueUniverseProvider _iQFeedDataProvider; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + _iQFeedDataProvider = new IQFeedDataProvider(); + } + + private static IEnumerable LookUpSymbolsTestParameters + { + get + { + yield return new TestCaseData(Symbol.Create(Futures.Energy.NaturalGas, SecurityType.Future, Market.NYMEX)); + yield return new TestCaseData(Symbol.Create(Futures.Indices.NASDAQ100EMini, SecurityType.Future, Market.CME)); + yield return new TestCaseData(Symbol.CreateCanonicalOption(Symbols.AAPL)); + yield return new TestCaseData(Symbol.CreateCanonicalOption(Symbols.SPY)); + } + } + + [Test, TestCaseSource(nameof(LookUpSymbolsTestParameters))] + public void LookUpSymbols(Symbol symbol) + { + var symbols = _iQFeedDataProvider.LookupSymbols(symbol, true, default).ToList(); + Assert.IsNotEmpty(symbols); + Assert.Greater(symbols.Count, 1); + Assert.AreEqual(symbols.Count, symbols.Distinct().Count()); + } +} diff --git a/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs b/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs index cac456a..132d3fc 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs @@ -20,8 +20,8 @@ using QuantConnect.Util; using QuantConnect.Tests; using QuantConnect.Securities; -using System.Collections.Generic; using QuantConnect.Data.Market; +using System.Collections.Generic; namespace QuantConnect.Lean.DataSource.IQFeed.Tests { @@ -30,8 +30,8 @@ public class IQFeedHistoryProviderTests { private IQFeedDataProvider _historyProvider; - [SetUp] - public void SetUp() + [OneTimeSetUp] + public void OneTimeSetUp() { _historyProvider = new IQFeedDataProvider(); _historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, null, null, null)); @@ -61,13 +61,23 @@ internal static IEnumerable HistoricalTestParameters yield return new TestCaseData(AAPL, Resolution.Hour, TickType.Quote, TimeSpan.FromDays(180), true); yield return new TestCaseData(AAPL, Resolution.Daily, TickType.Quote, TimeSpan.FromDays(365), true); - // TickType.OpenInterest is not maintained - yield return new TestCaseData(AAPL, Resolution.Tick, TickType.OpenInterest, TimeSpan.FromMinutes(5), true); + // Future + var nasdaq100EMini = Symbol.CreateFuture(Futures.Indices.NASDAQ100EMini, Market.CME, new DateTime(2025, 09, 19)); + yield return new TestCaseData(nasdaq100EMini, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(180), false); + yield return new TestCaseData(nasdaq100EMini, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(180), false); + yield return new TestCaseData(nasdaq100EMini, Resolution.Minute, TickType.Trade, TimeSpan.FromDays(10), false); + yield return new TestCaseData(nasdaq100EMini, Resolution.Second, TickType.Trade, TimeSpan.FromMinutes(10), false); + yield return new TestCaseData(nasdaq100EMini, Resolution.Tick, TickType.Trade, TimeSpan.FromMinutes(5), false); + + yield return new TestCaseData(AAPL, Resolution.Tick, TickType.OpenInterest, TimeSpan.FromMinutes(5), true).SetDescription("TickType.OpenInterest is not maintained"); // Not supported Security Types yield return new TestCaseData(Symbol.Create("SPX.XO", SecurityType.Index, Market.CBOE), Resolution.Tick, TickType.Trade, TimeSpan.FromMinutes(5), true); - yield return new TestCaseData(Symbol.CreateFuture("@ESGH24", Market.CME, new DateTime(2024, 3, 21)), Resolution.Tick, TickType.Trade, TimeSpan.FromMinutes(5), true); + var naturalGasAug2025 = Symbol.CreateFuture(Futures.Energy.NaturalGas, Market.NYMEX, new DateTime(2025, 08, 27)); + yield return new TestCaseData(naturalGasAug2025, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(100), false); + + yield return new TestCaseData(Symbol.CreateCanonicalOption(Symbols.AAPL), default, default, default, true).SetDescription("Canonical symbol requests are unsupported and return null."); } } @@ -99,7 +109,7 @@ internal static void AssertTicksHaveAppropriateTickType(Resolution resolution, T case (Resolution.Tick, TickType.Trade): Assert.IsTrue(historyResponse.Any(x => x.Ticks.Any(xx => xx.Value.Count > 0 && xx.Value.Any(t => t.TickType == TickType.Trade)))); break; - }; + } } internal static void AssertHistoricalDataResponse(Resolution resolution, List historyResponse) @@ -126,17 +136,22 @@ internal static void AssertHistoricalDataResponse(Resolution resolution, List GetFutureSymbolsTestCases() + { + // Natural gas futures expire the month previous to the contract month: + // Expiry: August -> Contract month: September (U) + yield return new TestCaseData("QNGU25", Symbol.CreateFuture(Futures.Energy.NaturalGas, Market.NYMEX, new DateTime(2025, 08, 27))); + // Expiry: December 2025 -> Contract month: January (U) 2026 (26) + yield return new TestCaseData("QNGF26", Symbol.CreateFuture(Futures.Energy.NaturalGas, Market.NYMEX, new DateTime(2025, 12, 29))); + + // BrentLastDayFinancial futures expire two months previous to the contract month: + // Expiry: August -> Contract month: October (V) + yield return new TestCaseData("QBZV25", Symbol.CreateFuture(Futures.Energy.BrentLastDayFinancial, Market.NYMEX, new DateTime(2025, 08, 29))); + // Expiry: November 2025 -> Contract month: January (F) 2026 (26) + yield return new TestCaseData("QBZF26", Symbol.CreateFuture(Futures.Energy.BrentLastDayFinancial, Market.NYMEX, new DateTime(2025, 11, 28))); + // Expiry: December 2025 -> Contract month: February (G) 2026 (26) + yield return new TestCaseData("QBZG26", Symbol.CreateFuture(Futures.Energy.BrentLastDayFinancial, Market.NYMEX, new DateTime(2025, 12, 31))); + + yield return new TestCaseData("@NQU25", Symbol.CreateFuture(Futures.Indices.NASDAQ100EMini, Market.CME, new DateTime(2025, 09, 19))); + + yield return new TestCaseData("@ADU25", Symbol.CreateFuture(Futures.Currencies.AUD, Market.CME, new DateTime(2025, 09, 15))) + .SetDescription("IQFeed returns 'AD' for Australian Dollar; Lean expects '6A'. Verifies symbol mapping from 'IQFeed-symbol-map.json'."); + + yield return new TestCaseData("@BOU25", Symbol.CreateFuture(Futures.Grains.SoybeanOil, Market.CBOT, new DateTime(2025, 09, 12))) + .SetDescription("Brokerage uses 'BO' for Soybean Oil; Lean maps it as 'ZL'. Validates symbol translation via 'IQFeed-symbol-map.json'."); + } + + [TestCaseSource(nameof(GetFutureSymbolsTestCases))] + public void ConvertsFutureSymbolRoundTrip(string brokerageSymbol, Symbol leanSymbol) + { + var actualBrokerageSymbol = _dataQueueUniverseProvider.GetBrokerageSymbol(leanSymbol); + var actualLeanSymbol = _dataQueueUniverseProvider.GetLeanSymbol(brokerageSymbol, default, default); + + Assert.AreEqual(brokerageSymbol, actualBrokerageSymbol); + Assert.AreEqual(leanSymbol, actualLeanSymbol); + } + + [TestCase("NYMEX_GBX", "QQA", "QA")] + [TestCase("NYMEX_GBX", "QQAN25", "QAN25")] + public void NormalizeFuturesTickerRemovesFirstQForNymexGbxExchange(string exchange, string brokerageSymbol, string expectedNormalizedFutureSymbol) + { + var actualNormalizedFutureSymbol = IQFeedDataQueueUniverseProvider.NormalizeFuturesTicker(exchange, brokerageSymbol); + Assert.AreEqual(expectedNormalizedFutureSymbol, actualNormalizedFutureSymbol); + } } } diff --git a/QuantConnect.IQFeed/IQFeed-symbol-map.json b/QuantConnect.IQFeed/IQFeed-symbol-map.json index 40d411d..d59f843 100644 --- a/QuantConnect.IQFeed/IQFeed-symbol-map.json +++ b/QuantConnect.IQFeed/IQFeed-symbol-map.json @@ -1,2 +1,2 @@ /* This is an auto-generated file that contains mappings from IQFeed own naming to original symbols defined by respective exchanges. Delete this file if you want it to be regenerated (long operation). */ -{"1AEX":"1FT","1MT":"1FC","2AEX":"2FT","2AF":"2AF","2AW":"2AW","2EF":"2EF","2EW":"2EW","2MT":"2FC","3AF":"3AF","3AW":"3AW","4AEX":"4FT","4MT":"4FC","67T":"67T","8EF":"8EF","10Y":"10Y","20U":"20U","25U":"25U","2GT":"2GT","2VT":"2VT","2YY":"2YY","30U":"30U","30Y":"30Y","35U":"35U","3N":"Z3N","40U":"40U","45U":"45U","50U":"50U","55U":"55U","5YY":"5YY","60U":"60U","65U":"65U","6EB":"6EB","A2T":"A2T","ACD":"ACD","AC":"EH","AD":"6A","ADT":"ADT","AE":"AW","AFR":"AFR","AFT":"AFT","AJY":"AJY","ANE":"ANE","AQT":"AQT","AR":"AR","ART":"ART","AS":"AS","ASI":"ASI","ASN":"ASN","ASR":"ASR","AST":"AST","AUW":"AUW","AWN":"AWN","AWT":"AWT","B1S":"B1S","BCX":"BCX","BF":"SS","BGT":"BGT","BIB":"BIB","BIO":"BIO","BIT":"BIT","BLK":"BLK","BNB":"BNB","BOB":"BOB","BO":"ZL","BOT":"BOT","BP":"6B","BR":"6L","BSB":"BSB","BTB":"BTB","BTC":"BTC","BTE":"BTE","BT":"BOS","BUB":"BUB","C3S":"C3S","CAD":"CAD","CB":"CB","CC":"CC","CCT":"CCT","CD":"6C","C":"ZC","CH":"CHI","CHL":"CHL","CHP":"CHP","CJY":"CJY","CKO":"CZK","CKS":"CKS","CNH":"CNH","CPO":"CPO","CPV":"CPV","CSC":"CSC","CT":"CT","CTT":"CTT","CU":"CUS","CWR":"CWR","D0":"D0","D1":"D1","D1X":"D1X","D1Z":"D1Z","D2":"D2","D4":"D4","D4X":"D4X","D4Z":"D4Z","DA":"DC","DC":"WDC","DE":"DEN","DFN":"DFN","DGS":"DGS","DGT":"DGT","DK":"GDK","DRS":"DRS","DRT":"DRT","DVE":"DVE","DVT":"DVT","DX":"DX","DXT":"DXT","DY":"DY","E1S":"E1S","E3G":"E3G","E3T":"E3T","EAD":"EAD","EAT":"EAT","EBN":"EBN","EBR":"EBR","ECD":"ECD","ECW":"ECW","ECZ":"ECK","EDW":"EDW","EGT":"EGT","EHU":"EHF","EIC":"EIC","EID":"EID","EI":"EI","EIT":"EIT","EIY":"EIY","EMB":"EMB","EMD":"EMD","EMT":"EMT","ENB":"ENB","ENK":"ENK","ENY":"ENY","EPL":"EPZ","ESG":"ESG","ES":"ES","ESK":"ESK","ESQ":"ESQ","ESR":"ESR","EST":"EST","ESX":"ESX","ETB":"ETB","ETC":"ETC","ETE":"ETE","ETH":"ETH","ETR":"ETR","ETW":"ETW","EU9":"EU9","EU":"6E","EUS":"EUS","EWV":"EWV","EYB":"EYB","EYT":"EYT","EZ":"EZ","F1S":"F1S","FF":"ZQ","FIT":"FIT","FIX":"FIX","FNG":"FNG","FOB":"FOB","FOL":"FOL","FR":"SFR","FT1":"FT1","FT5":"FT5","FTB":"FTB","FTC":"FTC","FTT":"FTT","FTU":"FTU","FTW":"FTW","FV":"ZF","FYN":"FYN","FYT":"FYT","FYW":"FYW","FZE":"FZE","G0":"G0","G1":"G1","G1K":"G1K","G1N":"G1N","G2":"G2","G4":"G4","G4K":"G4K","G4N":"G4N","G6":"G6","G6K":"G6K","G6N":"G6N","G6X":"G6X","G6Z":"G6Z","GDT":"GDT","GF":"GF","GFT":"GFT","GIE":"GIE","GI":"GD","GIT":"GIT","GN":"GN","H2O":"H2O","H6":"H6","H6X":"H6X","H6Z":"H6Z","H7":"H7","H7X":"H7X","H7Z":"H7Z","HE":"HE","HET":"HET","HFO":"HUF","HQ":"HQ","HRE":"HR","HR":"HR","HRX":"HRX","HRZ":"HRZ","HW":"HW","HWX":"HWX","HWZ":"HWZ","HY":"HY","IBH":"IBH","IBI":"IBI","IBT":"IBT","IBV":"IBV","IC":"IC","IH":"IH","IL":"ILS","ILS":"ILS","IP":"IP","IPO":"IPO","IPT":"IPT","IS":"IS","IST":"IST","IW":"IW","JE":"J7","JPP":"JPP","JY":"6J","K0":"K0","K1":"K1","K2":"K2","K3":"K3","K4":"K4","K5":"K5","K6":"K6","K6K":"K6K","K6N":"K6N","K6S":"K6S","K7":"K7","K7K":"K7K","K7N":"K7N","KAT":"KAT","KAU":"KAU","KC7":"KC7","KC":"KC","KCT":"KCT","KEJ":"KEJ","KEO":"KEO","KEP":"KEP","KGB":"KGB","KGT":"KGT","KJ":"KJ","KJT":"KJT","KKT":"KKT","KMF":"KMF","KMP":"KMP","KNS":"KNS","KOL":"KOL","KOT":"KOT","KP":"KP","KPK":"KPK","KPN":"KPN","KQ":"KQ","KRA":"KRA","KRH":"KRK","KR":"KR","KRK":"KRK","KRN":"KRN","KRW":"KRW","KRZ":"KRZ","KS":"KS","KSN":"KSN","KSV":"KSV","KTT":"KTT","KWB":"KW","KW":"KE","KWK":"KWK","KWN":"KWN","KWT":"KET","KXD":"KX","KY":"KY","KZS":"KZS","KZT":"KZT","KZX":"KZX","KZY":"KZY","LA":"LAX","LBR":"LBR","LE":"LE","LET":"LET","LP":"LP","LPX":"LPX","LPZ":"LPZ","LV":"LAV","M2K":"M2K","M6A":"M6A","M6B":"M6B","M6C":"M6C","M6E":"M6E","M6J":"M6J","M6S":"M6S","MB1":"MB1","MBT":"MBT","MCD":"MCD","MCE":"MCE","MCL":"MCL","MCV":"MCU","MCX":"MCX","ME":"E7","MES":"MES","MET":"MET","MEU":"MEU","MFC":"MFC","MFS":"MFS","MFU":"MFU","MGE":"MGE","MIB":"MIB","MI":"MIA","MIN":"MIN","MIR":"MIR","MKC":"MKC","MLE":"MLE","MMC":"MMC","MME":"MME","MML":"MML","MMM":"MMM","MMN":"MMN","MMR":"MMR","MMW":"MMW","MNH":"MNH","MNQ":"MNQ","MPA":"MPA","MPC":"MPC","MP":"MP","MPP":"MPP","MPT":"MPT","MPU":"MPU","MRG":"MRG","MS1":"MS1","MS4":"MS4","MS5":"MS5","MS6":"MS6","MS7":"MS7","MS8":"MS8","MS9":"MS9","MSC":"MSC","MSF":"MSF","MT1":"MT1","MT3":"MT3","MUC":"MUC","MUN":"MUN","MUS":"MUS","MW":"MWE","MWL":"MWL","MWS":"MWS","MY1":"MY1","MY2":"MY2","MY3":"MY3","MY4":"MY4","MY5":"MY5","MY7":"MY7","MYB":"MYB","MYM":"MYM","N1S":"N1S","NAA":"NAA","NBY":"NBY","NCB":"NCB","NCF":"NCF","NDA":"NDA","NE":"6N","NF":"GNF","NIB":"NIB","NIT":"NIT","NIY":"NIY","NJ":"NJ","NKD":"NKD","NKT":"NKT","NOB":"NOB","NOK":"NOK","NOL":"NOL","NON":"NON","NQB":"NQT","NQ":"NQ","NQQ":"NQQ","NQX":"NQX","NT":"NT","NTW":"NTW","NUB":"NUB","NY":"NYM","NYW":"NYW","O":"ZO","OJ":"OJ","OJT":"OJT","OPF":"OPF","PAC":"PAC","PC":"PC","PJY":"PJY","PK":"PK","PLE":"PLE","PLN":"PLN","PLZ":"PLN","POG":"POG","PRK":"PRK","PSF":"PSF","PS":"PS","PX":"6M","PY":"SY","PZ":"PZ","QA":"QA","QC2":"QC2","QC3":"QC3","QC4":"QC4","QC6":"QC6","QC7":"QC7","QC8":"QC8","QCC":"QCC","QCN":"QCN","QCW":"QCW","QKC":"QKC","QM2":"QM2","QM3":"QM3","QM4":"QM4","QM5":"QM5","QM6":"QM6","QO2":"QO2","QO3":"QO3","QO4":"QO4","QO5":"QO5","QO6":"QO6","QS0":"QS0","QS1":"QS1","QS2":"QS2","QS3":"QS3","QS5":"QS5","QS8":"QS8","QS9":"QS9","QW2":"QW2","QW3":"QW3","QW6":"QW6","QX5":"QX5","R1":"RS1","R1T":"R1T","R2G":"R2G","R2V":"R2V","RA":"6Z","RB":"RMB","RCT":"RST","RDA":"RDA","RE":"RME","RET":"RET","REX":"REX","RFD":"RFD","RF":"RF","RFI":"RFI","RKT":"RKT","RLT":"RLT","RP":"RP","RR":"ZR","RSD":"RSD","RS":"RS","RSG":"RSG","RSI":"RSI","RST":"RGT","RSV":"RSV","RTQ":"RTQ","RTX":"RTX","RTY":"RTY","RU":"6R","RUT":"RVT","RX":"RX","RY":"RY","S1S":"S1S","S7C":"S7C","SAS":"SAS","SB":"SB","SBT":"SBT","SDA":"SDA","SD":"SDG","SDI":"SDI","SEK":"SEK","S":"ZS","SF":"6S","SFI":"SF","SG":"SG","SGT":"SGT","SIR":"SIR","SJY":"SJY","SMC":"SMC","SM":"ZM","SMT":"SMT","SON":"SON","SOT":"SBT","SOX":"SOX","SQ2":"SQ2","SQ5":"SQ5","SR1":"SR1","SR3":"SR3","SU":"SU","SUT":"SUT","SWT":"SWT","SXB":"SXB","SXI":"SXI","SXO":"SXO","SXR":"SXR","SXT":"SXT","T1S":"T1S","TAF":"TAF","TBF":"TBF","TBM":"TBM","TBT":"TBT","TEX":"TEX","TFY":"TFY","THW":"THW","TIE":"TIE","TN":"TN","TNT":"TNT","TOB":"TOB","TOF":"TOF","TOS":"SOT","TOU":"TOU","TOW":"TOW","TOX":"TOX","TPB":"TPB","TPD":"TPD","TPT":"TPT","TPY":"TPY","TRB":"TRB","TRI":"TRI","TRL":"TRL","TRM":"TRM","TRW":"TRW","TTW":"TTW","TUB":"TUB","TUF":"TUF","TU":"ZT","TUL":"TUL","TUT":"TUT","TUX":"TUX","TUY":"TUN","TWB":"TWB","TWE":"TWE","TWU":"TWU","TWW":"TWW","TY":"ZN","TYT":"TYT","TYW":"TYW","TYX":"TYX","UB":"UB","UBT":"UBT","UFB":"UFB","UFE":"UFE","UFV":"UFV","UNO":"UNO","US":"ZB","USS":"USS","VC":"VC","VMT":"VMT","VU":"VU","VX1":"VX1","VX2":"VX2","VX4":"VX4","VX5":"VX5","VX":"VX","VXM":"VXM","W":"ZW","WMK":"WMK","WQ6":"WQ6","XAB":"XAB","XAE":"XAE","XAF":"XAF","XAI":"XAI","XAK":"XAK","XAP":"XAP","XAR":"XAR","XAU":"XAU","XAV":"XAV","XAY":"XAY","XAZ":"XAZ","XCT":"XVT","XET":"XET","XFT":"XFT","XIT":"XIT","XKT":"XKT","XMT":"XBT","XPT":"XPT","XTT":"XRT","XUT":"XUT","XVT":"VXT","XYT":"XYT","XZT":"XZT","YA":"YA","YC":"XC","YIA":"YIA","YIB":"YIB","YIC":"YIC","YID":"YID","YIE":"YIE","YII":"YII","YIL":"YIL","YIO":"YIO","YIT":"YIT","YIW":"YIW","YIY":"YIY","YK":"XK","YM":"YM","YMT":"YMT","YMX":"YMX","YW":"XW","YZ":"YZ","Z3W":"Z3W","ZAR":"ZAR","ZBT":"ZBT","ZBW":"ZBW","ZCT":"ZCT","ZFT":"ZFT","ZJ":"ZJ","ZLT":"ZLT","ZMT":"ZMT","ZNS":"ZNS","ZR":"ZR","ZTT":"ZTT","ZTW":"ZTW","ZWC":"ZWC","ZWT":"ZWT","AEX":"FTI","AFB":"UB","ALJ":"ALS","ALM":"ALM","AM":"AM","ARB":"ARB","ARS":"ARS","ATX":"ATX","AUD":"AUD","AXA":"AXA","AXF":"AXF","AXJ":"AXJ","AXL":"AXL","AXM":"AXM","AXQ":"AXQ","AXS":"AXS","AXV":"AXV","BB":"JB","BBN":"BB","BD":"GBL","BFX":"BXF","BGI":"BGI","BL":"GBM","BN":"BN","BON":"BON","BQ":"BQ","BRI":"BRI","BS":"BS","BTM":"BTM","BTP":"BTP","BTS":"BTS","BTU":"BTU","BV":"BV","BX":"GBX","C1L":"C1L","C3F":"C3F","C5F":"C5F","C7F":"C7F","CAG":"CAG","CA":"EMA","CCM":"CCM","CCO":"CCO","CEE":"CEE","CEN":"CEN","CEX":"CEX","CGR":"CGR","CHF":"CHF","CIN":"CIN","CLI":"CLI","CLP":"CLP","CN":"CN","CNI":"CNY","CO":"CON","COP":"COP","COR":"COR","CPE":"CPE","CPR":"CPR","CRD":"WBS","CSO":"CSO","CWF":"CWF","CXA":"CXA","CXB":"CXB","CXE":"CXE","CXI":"CXI","CXL":"CXL","CXP":"CXP","CXR":"CXR","CXS":"CXS","CXT":"CXT","D3D":"D3D","DDI":"DDI","DIJ":"DI1","DIV":"DIV","DOL":"DOL","DVD":"DVD","DXD":"DXD","DXM":"DXM","DXS":"DXS","EA":"EA","EAL":"MEA","EAN":"BEA","EBD":"EBD","EBE":"EU3","EB":"BRN","EBF":"EBF","EBI":"EBD","EBT":"BRT","ECX":"ECX","EDV":"EDV","EED":"EED","EE":"EE","EFD":"EFD","EF":"AFR","EFQ":"AFQ","EFS":"AFS","EFY":"AFY","EMA":"EMA","EPE":"EPE","EPO":"EPO","EPR":"EPR","ESA":"ESG","ESC":"ESC","ESE":"ESE","ESF":"ESF","ESJ":"ESJ","ESL":"ESL","ESM":"ESM","ESN":"ESN","ESO":"ESO","ESP":"ESP","ESS":"ESS","ESU":"ESU","ESV":"ESV","ESW":"ESW","ESZ":"ESZ","ETD":"ETD","ET":"ATW","ETQ":"ATQ","ETS":"ATS","ETY":"ATY","EU3":"EU3","EUD":"EUD","EUR":"EUR","EXD":"EXD","EX":"ESX","EXZ":"XZ","FCG":"FCG","FCT":"FCT","FID":"FID","FIN":"FIN","FMX":"FMX","FMY":"FMY","FNA":"FNA","FND":"FND","FN":"GWM","FNQ":"NGQ","FNS":"NGS","FNY":"FNY","FOX":"FOX","FPD":"FPD","FPH":"FPH","FRC":"FRC","FTH":"FTH","FVN":"FVN","FVS":"FVS","G":"G02","GAS":"G","GB3":"GB3","GBH":"GBH","GBO":"GBO","GBP":"GBP","GBQ":"GBP","GBW":"GBW","GDI":"GDI","GDS":"GDS","GDV":"GDV","GGI":"GGI","GLD":"GLD","GOL":"GOL","GT":"GT","GWT":"GWT","GWY":"GWY","GX":"GX","H":"H","HOS":"HOS","HOU":"HOU","HSI":"HSI","HT":"HT","HWF":"HWF","I":"I02","IB":"IB","ICF":"ICF","IE":"I","IET":"IT","IHO":"UHO","IND":"IND","IRB":"UHU","IRBT":"LRT","IR":"IR","ISE":"ISE","ISP":"ISP","JG":"JG","JPI":"JPY","JRT":"JRT","KAN":"KAN","KLI":"KLI","KPO":"CPO","LCE":"LCE","LCP":"LCP","LF":"Z","LG":"R","LGT":"RT","LHT":"LHT","LHW":"USW","LIV":"USO","LJN":"USP","LLR":"FEF","LLS":"FEO","LRC":"RC","LRCT":"RCT","LU":"TWS","LY":"Y2","M":"M70","MAE":"MAE","MAS":"MAS","MAW":"MAW","MAX":"MFA","MBE":"MBE","MBZ":"MBZ","MC2":"MCP","MCH":"MCH","MDK":"MDK","MDM":"MDM","MEA":"MEA","MEE":"MEE","MEF":"MEF","MEL":"MEL","MEM":"MEM","MEN":"MEN","MEP":"MEP","MFI":"MFI","MFR":"MFR","MGA":"MGA","MGC":"MGC","MGS":"MGS","MIX":"MIX","MJP":"MJP","MKU":"MKU","MMY":"MMY","MNL":"MNL","MNO":"MNA","MNS":"MNS","MNW":"MNW","MSD":"MSD","MSG":"MSG","MSI":"MSI","MSJ":"MSJ","MSP":"MSP","MSR":"MSM","MSS":"MSS","MST":"MST","MSU":"MSU","MSW":"MSW","MSZ":"MSZ","MT":"FCE","MTH":"MTH","MTW":"MTW","MWA":"WM","MWB":"MWB","MWC":"MWC","MWD":"MWD","MWF":"MWF","MWH":"MWH","MWI":"MWI","MWN":"MWN","MWO":"MWO","MWP":"MWP","MWQ":"MWQ","MWR":"MWR","MWT":"MWT","MXB":"MXB","MXG":"MXG","MXN":"MXN","MZA":"MZA","NAU":"NAU","NCQ":"NCQ","NCY":"NCY","NDI":"NDI","NEA":"NEA","NID":"NID","NN":"NK","NPH":"NPH","NTH":"NTH","NU":"NU","NXJ":"NXJ","NZD":"NZD","OAM":"OAM","OAT":"OAT","OE":"O","OFR1":"SF1","OFR3":"SF3","OIL":"OIL","OP":"P","PG":"ECO","PLA":"PLA","PM":"EBM","PNS":"PN","POL":"POL","PQ":"PQ","PSI":"PSI","PVF":"PVF","PV":"PV","PWF":"PWF","QC":"C","QCT":"LCT","QK":"T","QUME":"UME","QW":"W","QWT":"WT","RED":"RED","RES":"RES","SA":"ESA","SBD":"SBD","SCE":"SCE","SCI":"SCI","SCP":"SCP","SDX":"SDX","SED":"SED","SEE":"STE","SEG":"SEG","SID":"SID","SIN":"ESI","SJC":"SJC","SJ":"EST","SK":"ESH","SLC":"SLC","SLI":"SLI","SLS":"SLS","SLV":"SLV","SMD":"SMD","SMM":"SMM","SMS":"SMS","SMX":"SMX","SND":"ND","SNS":"NS","SO3":"SO3","SOA":"SOA","SOY":"SOY","SPI":"AP","SQ":"STX","SS":"SGP","SSX":"SSX","ST3":"ST3","STD":"STD","STF":"STF","STG":"STG","STI":"ST","STJ":"STJ","STL":"STL","STM":"STM","STN":"STI","STO":"STO","STP":"STP","STQ":"STQ","STS":"STS","STU":"STU","STW":"STW","SUD":"SUD","SUN":"SUN","SUS":"SUS","SVF":"SVF","SWF":"SWF","SW":"SMI","SXE":"SXE","SX":"ESB","SY":"ESY","TA":"STB","TC":"STC","TDD":"TDD","TD":"TDX","TF":"STA","TIN":"TIN","TJ":"STH","TL":"STV","TM":"STN","TP":"STZ","TR":"STR","TT":"STT","TUK":"TUK","TWN":"TWN","TX":"STY","UAA":"UAA","UAL":"UAL","UAM":"UAM","UAQ":"UAQ","UAS":"UAS","UAV":"UAV","UBL":"UBL","UBQ":"UBQ","U":"U","UKA":"UKA","UKD":"UKD","UNF":"UNF","UPL":"UPL","UPO":"UPO","US3":"US3","UT":"UT","UZQ":"UZQ","WAW":"WK","WBT":"WBT","WDO":"WDO","WEA":"WEA","WEU":"WEU","WIN":"WIN","WMA":"WMA","WPD":"WPD","WTO":"WTO","XFC":"XFC","XG":"DAX","XGL":"XGL","XSF":"XSF","XT":"XT","XXE":"XXE","XXP":"XXP","XXS":"XXS","YMA":"YMA","YT":"YT"} \ No newline at end of file +{"1AEX":"1FT","1MT":"1FC","2AEX":"2FT","2EW":"2EW","2MT":"2FC","4AEX":"4FT","4MT":"4FC","67T":"67T","10Y":"10Y","1D0":"1D0","1D1":"1D1","1D2":"1D2","1D4":"1D4","1G6":"1G6","1H0":"1H0","1H1":"1H1","1H2":"1H2","1H3":"1H3","1H4":"1H4","1H5":"1H5","1H6":"1H6","1H7":"1H7","1HQ":"1HQ","1HR":"1HR","1HS":"1HS","1HW":"1HW","1LP":"1LP","20U":"20U","25U":"25U","2GT":"2GT","2K0":"2K0","2K1":"2K1","2K2":"2K2","2K3":"2K3","2K4":"2K4","2K5":"2K5","2K6":"2K6","2K7":"2K7","2KP":"2KP","2KQ":"2KQ","2KR":"2KR","2KS":"2KS","2KW":"2KW","2VT":"2VT","2YY":"2YY","30U":"30U","30Y":"30Y","33K":"33K","35U":"35U","3G0":"3G0","3G1":"3G1","3G2":"3G2","3G4":"3G4","3G6":"3G6","3K0":"3K0","3K1":"3K1","3K2":"3K2","3K3":"3K3","3K4":"3K4","3K5":"3K5","3K6":"3K6","3K7":"3K7","3KP":"3KP","3KQ":"3KQ","3KS":"3KS","3KW":"3KW","3N":"Z3N","40U":"40U","45U":"45U","4D0":"4D0","4D1":"4D1","4D2":"4D2","4D4":"4D4","4G6":"4G6","4H0":"4H0","4H1":"4H1","4H2":"4H2","4H3":"4H3","4H4":"4H4","4H5":"4H5","4H6":"4H6","4H7":"4H7","4HQ":"4HQ","4HR":"4HR","4HS":"4HS","4HW":"4HW","4LP":"4LP","50U":"50U","55U":"55U","5YY":"5YY","60U":"60U","65U":"65U","6EB":"6EB","70U":"70U","A2R":"A2R","A2T":"A2T","ABB":"ABB","ACD":"ACD","AD":"6A","ADR":"ADR","ADT":"ADT","AE":"AW","AFR":"AFR","AFT":"AFT","AHB":"AHB","AJY":"AJY","AMB":"AMB","ANE":"ANE","AQR":"AQR","AQT":"AQT","AR":"AR","ARR":"ARR","ART":"ART","AS":"AS","ASI":"ASI","ASN":"ASN","ASR":"ASR","AST":"AST","ATB":"ATB","AUW":"AUW","AWN":"AWN","AWT":"AWT","B1S":"B1S","BAG":"BAG","BAT":"BAT","BCX":"BCX","BEN":"BEN","BET":"BET","BF":"SS","BGR":"BGR","BIO":"BIO","BIT":"BIT","BLI":"BLI","BLK":"BLK","BLT":"BLT","BME":"BME","BMT":"BMT","BNB":"BNB","BO":"ZL","BOT":"BOT","BPE":"BPE","BP":"6B","BPR":"BPR","BPT":"BPT","BR":"6L","BST":"BST","BTB":"BTB","BTC":"BTC","BTE":"BTE","BT":"BOS","BTG":"BGT","C3S":"C3S","CAD":"CAD","CB":"CB","CCB":"CCT","CC":"CC","CCI":"CCI","CCT":"CCT","CD":"6C","C":"ZC","CH":"CHI","CHP":"CHP","CJY":"CJY","CKO":"CZK","CKS":"CKS","CNH":"CNH","CPO":"CPO","CPV":"CPV","CSC":"CSC","CT":"CT","CTT":"CTT","CU":"CUS","CVB":"CVB","CW1":"CW1","CWD":"CWD","CWR":"CWR","D0":"D0","D1":"D1","D1X":"D1X","D1Z":"D1Z","D2":"D2","D4":"D4","D4X":"D4X","D4Z":"D4Z","DA":"DC","DC":"WDC","DE":"DEN","DFN":"DFN","DGS":"DGS","DGT":"DGT","DHB":"DHB","DHY":"DHY","DK":"GDK","DRS":"DRS","DRT":"DRT","DVE":"DVE","DVT":"DVT","DX":"DX","DXT":"DXT","DY":"DY","DYT":"DYT","E1S":"E1S","E3G":"E3G","E3T":"E3T","EAD":"EAD","EBM":"EBM","EBR":"EBR","ECD":"ECD","ECZ":"ECK","EEM":"EEM","EGT":"EGT","EHU":"EHF","EI":"EI","EIT":"EIT","EMB":"EMB","EMD":"EMD","EMT":"EMT","EMV":"EMV","ENB":"ENB","ENK":"ENK","ENY":"ENY","ENZ":"ENZ","EPL":"EPZ","ESG":"ESG","ES":"ES","ESK":"ESK","ESQ":"ESQ","ESR":"ESR","EST":"EST","ESX":"ESX","ETB":"ETB","ETE":"ETE","ETH":"ETH","ETR":"ETR","EU9":"EU9","EU":"6E","EUS":"EUS","EWE":"EWE","EWF":"EWF","EWM":"EWM","EWT":"EWT","EYB":"EYB","EZ":"EZ","F1S":"F1S","FF":"ZQ","FFV":"FFV","FNG":"FNG","FR":"SFR","FT1":"FT1","FT5":"FT5","FTB":"FTB","FTC":"FTC","FTT":"FTT","FTU":"FTU","FV":"ZF","G":"G02","G0":"G0","G1":"G1","G1K":"G1K","G1N":"G1N","G2":"G2","G4":"G4","G4K":"G4K","G4N":"G4N","G6":"G6","G6K":"G6K","G6N":"G6N","G6X":"G6X","G6Z":"G6Z","GDT":"GDT","GF":"GF","GFT":"GFT","GIE":"GIE","GI":"GD","GIT":"GIT","GN":"GN","H2O":"H2O","H6":"H6","H6X":"H6X","H6Z":"H6Z","H7":"H7","H7X":"H7X","H7Z":"H7Z","HE":"HE","HET":"HET","HFO":"HUF","HQ":"HQ","HRE":"HR","HR":"HR","HRS":"HRS","HRX":"HRX","HRZ":"HRZ","HW":"HW","HWX":"HWX","HWZ":"HWZ","HYB":"HYB","HY":"HY","HYT":"HYT","IBH":"IBH","IBI":"IBI","IBV":"IBV","IDR":"IDR","IEM":"IEM","IL":"ILS","ILS":"ILS","IPO":"IPO","IPT":"IPT","IQB":"IQB","IQT":"IQT","IST":"IST","JE":"J7","JPP":"JPP","JY":"6J","K0":"K0","K1":"K1","K2":"K2","K3":"K3","K4":"K4","K5":"K5","K6":"K6","K6K":"K6K","K6N":"K6N","K6S":"K6S","K7":"K7","K7K":"K7K","K7N":"K7N","KAT":"KAT","KAU":"KAU","KC7":"KC7","KC":"KC","KCT":"KCT","KEJ":"KEJ","KEO":"KEO","KEP":"KEP","KGB":"KGB","KGT":"KGT","KJ":"KJ","KJT":"KJT","KKT":"KKT","KMF":"KMF","KMP":"KMP","KNS":"KNS","KOL":"KOL","KOT":"KOT","KP":"KP","KPK":"KPK","KPN":"KPN","KQ":"KQ","KRA":"KRA","KRH":"KRK","KR":"KR","KRK":"KRK","KRN":"KRN","KRW":"KRW","KRZ":"KRZ","KS":"KS","KSN":"KSN","KSV":"KSV","KTT":"KTT","KWB":"KW","KWD":"KWD","KW":"KE","KWK":"KWK","KWN":"KWN","KWT":"KET","KXD":"KX","KY":"KY","KZS":"KZS","KZT":"KZT","KZX":"KZX","KZY":"KZY","LA":"LAX","LBR":"LBR","LE":"LE","LET":"LET","LP":"LP","LPX":"LPX","LPZ":"LPZ","LV":"LAV","M2K":"M2K","M6A":"M6A","M6B":"M6B","M6E":"M6E","MBT":"MBT","MCD":"MCD","MCE":"MCE","MCL":"MCL","MCV":"MCU","MCX":"MCX","ME":"E7","MES":"MES","MET":"MET","MEU":"MEU","MFC":"MFC","MFS":"MFS","MFU":"MFU","MFV":"MFV","MGE":"MGE","MIB":"MIB","MIE":"MIE","MI":"MIA","MIN":"MIN","MIR":"MIR","MIU":"MIU","MJY":"MJY","MKC":"MKC","MLE":"MLE","MMC":"MMC","MME":"MME","MML":"MML","MMM":"MMM","MMN":"MMN","MMR":"MMR","MMW":"MMW","MNH":"MNH","MNI":"MNI","MNK":"MNK","MNQ":"MNQ","MPA":"MPA","MP":"MP","MPP":"MPP","MPT":"MPT","MPU":"MPU","MRE":"MGE","MRG":"MRG","MSC":"MSC","MSF":"MSF","MSL":"MSL","MSU":"MSU","MTN":"MTN","MUC":"MUC","MUJ":"MUJ","MUK":"MUK","MUL":"MUL","MUN":"MUN","MUO":"MUO","MUP":"MUP","MUS":"MUS","MVM":"MVM","MVW":"MVW","MW":"MWE","MWL":"MWL","MWN":"MWN","MWS":"MWS","MXA":"MXA","MXE":"MXE","MXJ":"MXJ","MXP":"MXP","MYB":"MYB","MYM":"MYM","MZC":"MZC","MZL":"MZL","MZM":"MZM","MZS":"MZS","MZW":"MZW","N1S":"N1S","NAA":"NAA","NCF":"NCF","NDA":"NDA","NE":"6N","NF":"GNF","NIT":"NIT","NIY":"NIY","NJ":"NJ","NJY":"NJY","NKD":"NKD","NKT":"NKT","NOK":"NOK","NQB":"NQT","NQ":"NQ","NQQ":"NQQ","NQX":"NQX","NSK":"NSK","NT":"NT","NY":"NYM","NZC":"NZC","O":"ZO","OJ":"OJ","OJT":"OJT","OPF":"OPF","OSF":"OSF","PAC":"PAC","PAD":"PAD","PCD":"PCD","PC":"PC","PJY":"PJY","PK":"PK","PLE":"PLE","PLN":"PLN","PLZ":"PLN","PNK":"PNK","POG":"POG","PRK":"PRK","PSF":"PSF","PS":"PS","PSK":"PSK","PX":"6M","PY":"SY","PZ":"PZ","QA":"QA","QBT":"QBT","QC2":"QC2","QC3":"QC3","QC4":"QC4","QC6":"QC6","QC7":"QC7","QC8":"QC8","QCC":"QCC","QCN":"QCN","QCW":"QCW","QDF":"QDF","QDM":"QDM","QDO":"QDO","QEF":"QEF","QEM":"QEM","QET":"QET","QKC":"QKC","QM2":"QM2","QM3":"QM3","QM4":"QM4","QM5":"QM5","QM6":"QM6","QND":"QND","QNF":"QNF","QNM":"QNM","QO2":"QO2","QO3":"QO3","QO4":"QO4","QO5":"QO5","QO6":"QO6","QRF":"QRF","QRM":"QRM","QRT":"QRT","QS0":"QS0","QS1":"QS1","QS2":"QS2","QS3":"QS3","QS5":"QS5","QS9":"QS9","QSF":"QSF","QSM":"QSM","QSP":"QSP","QW2":"QW2","QW3":"QW3","QW6":"QW6","QX5":"QX5","R1":"RS1","R1T":"R1T","R2G":"R2G","R2V":"R2V","RA":"6Z","RB":"RMB","RCT":"RST","RDA":"RDA","RE":"RME","RET":"RET","REX":"REX","RFD":"RFD","RF":"RF","RFI":"RFI","RKT":"RKT","RLB":"RLB","RLT":"RLT","RNB":"RNB","RP":"RP","RR":"ZR","RSD":"RSD","RS":"RS","RSG":"RSG","RSI":"RSI","RSO":"RSO","RST":"RGT","RSV":"RSV","RTQ":"RTQ","RTX":"RTX","RTY":"RTY","RUT":"RVT","RX":"RX","RY":"RY","S1S":"S1S","S7C":"S7C","SAS":"SAS","SB":"SB","SBT":"SBT","SDA":"SDA","SD":"SDG","SDI":"SDI","SEK":"SEK","S":"ZS","SF":"6S","SFI":"SF","SGD":"SGD","SGT":"SGT","SG":"SG","SIR":"SIR","SJY":"SJY","SMC":"SMC","SM":"ZM","SMT":"SMT","SOL":"SOL","SON":"SON","SOT":"SBT","SOX":"SOX","SPR":"SPR","SPT":"SPT","SQ2":"SQ2","SQ5":"SQ5","SR1":"SR1","SR3":"SR3","SUT":"SUT","SU":"SU","SWT":"SWT","SXB":"SXB","SXI":"SXI","SXO":"SXO","SXR":"SXR","SXT":"SXT","SYP":"SYP","T1S":"T1S","TAF":"TAF","TBF":"TBF","TBM":"TBM","TBT":"TBT","TEM":"TEM","TET":"TET","TEX":"TEX","TFY":"TFY","THA":"THA","THW":"THW","TI3":"TI3","TIE":"TIE","TN":"TN","TNT":"TNT","TOB":"TOB","TOF":"TOF","TOS":"SOT","TOU":"TOU","TOW":"TOW","TOX":"TOX","TPB":"TPB","TPD":"TPD","TPT":"TPT","TPY":"TPY","TRB":"TRB","TRI":"TRI","TRL":"TRL","TRM":"TRM","TRW":"TRW","TTW":"TTW","TUB":"TUB","TUF":"TUF","TU":"ZT","TUL":"TUL","TUT":"TUT","TUX":"TUX","TUY":"TUN","TWB":"TWB","TWE":"TWE","TWU":"TWU","TWW":"TWW","TY":"ZN","TYT":"TYT","TYW":"TYW","TYX":"TYX","UB":"UB","UBT":"UBT","UFB":"UFB","UFE":"UFE","UFV":"UFV","US":"ZB","USS":"USS","VA":"VA","VC":"VC","VMT":"VMT","VU":"VU","VX1":"VX1","VX2":"VX2","VX4":"VX4","VX5":"VX5","VX":"VX","VXM":"VXM","W":"ZW","WK1":"KW1","WMK":"WMK","WQ6":"WQ6","XAB":"XAB","XAE":"XAE","XAF":"XAF","XAI":"XAI","XAK":"XAK","XAP":"XAP","XAR":"XAR","XAU":"XAU","XAV":"XAV","XAY":"XAY","XAZ":"XAZ","XBT":"XBT","XCT":"XVT","XET":"XET","XEU":"XEU","XFT":"XFT","XIT":"XIT","XKT":"XKT","XLB":"XLB","XMT":"XBT","XNB":"XNB","XPT":"XPT","XRP":"XRP","XTT":"XRT","XUT":"XUT","XVT":"VXT","XY3":"XY3","XYR":"XYR","XYT":"XYT","XZT":"XZT","YA":"YA","YC":"XC","YIA":"YIA","YIB":"YIB","YIC":"YIC","YID":"YID","YIE":"YIE","YII":"YII","YIL":"YIL","YIO":"YIO","YIT":"YIT","YIW":"YIW","YIY":"YIY","YK":"XK","YM":"YM","YMT":"YMT","YMX":"YMX","YW":"XW","YZ":"YZ","Z3W":"Z3W","ZBT":"ZBT","ZBW":"ZBW","ZCT":"ZCT","ZFT":"ZFT","ZJ":"ZJ","ZLT":"ZLT","ZMT":"ZMT","ZNS":"ZNS","ZR":"ZR","ZTT":"ZTT","ZTW":"ZTW","ZWC":"ZWC","ZWT":"ZWT","AEX":"FTI","AFB":"UB","AFS":"AFS","ALJ":"ALS","ALM":"ALM","AM":"AM","ARB":"ARB","ARS":"ARS","ASF":"ESF","ATX":"ATX","AUD":"AUD","AUT":"AUS","AXF":"AXF","AXJ":"AXJ","BB":"JB","BBN":"BB","BD":"GBL","BFX":"BXF","BGI":"BGI","BKS":"BKS","BL":"GBM","BMS":"BMS","BN":"BN","BON":"BON","BQ":"BQ","BRI":"BRI","BS":"BS","BTM":"BTM","BTP":"BTP","BTS":"BTS","BTU":"BTU","BV":"BV","BWS":"BCS","BX":"GBX","C3F":"C3F","C5F":"C5F","C7F":"C7F","CAG":"CAG","CA":"EMA","CAN":"CAN","CCM":"CCM","CCO":"CCO","CEE":"CEE","CEN":"CEN","CGR":"CGR","CHF":"CHF","CHL":"CHL","CIN":"CIN","CLI":"CLI","CLP":"CLP","CN":"CN","CO":"CON","COP":"COP","COR":"COR","CPE":"CPE","CPR":"CPR","CRD":"WBS","CSO":"CSO","CTO":"CTO","CWF":"CWF","CXA":"CXA","CXB":"CXB","CXE":"CXE","CXI":"CXI","CXL":"CXL","CXP":"CXP","CXR":"CXR","CXS":"CXS","CXT":"CXT","D3D":"D3D","DBI":"DBI","DDI":"DDI","DIJ":"DI1","DIV":"DIV","DOL":"DOL","DXM":"DXM","DXS":"DXS","EA":"EA","EAL":"MEA","EAN":"BEA","EBD":"EBD","EBE":"EU3","EB":"BRN","EBF":"EBF","EBI":"EBD","EBT":"BRT","ECX":"ECX","EDE":"EDE","EDV":"EDV","EDW":"EDW","EED":"EED","EE":"EE","EEU":"EEU","EFD":"EFD","EF":"AFR","EFQ":"AFQ","EFS":"AFS","EFY":"AFY","EID":"EID","EJP":"EJP","EMA":"EMA","EPE":"EPE","EPO":"EPO","EPR":"EPR","ER3":"ER3","ESA":"ESG","ESC":"ESC","ESE":"ESE","ESF":"ESF","ESJ":"ESJ","ESL":"ESL","ESM":"ESM","ESN":"ESN","ESO":"ESO","ESP":"ESP","ESS":"ESS","ESU":"ESU","ESV":"ESV","ESW":"ESW","ESZ":"ESZ","ETD":"ETD","ET":"ATW","ETQ":"ATQ","ETS":"ATS","ETY":"ATY","EU3":"EU3","EUD":"EUD","EUK":"EUK","EUP":"EUP","EUR":"EUR","EXD":"EXD","EX":"ESX","EXZ":"XZ","FCG":"FCG","FCT":"FCT","FID":"FID","FIN":"FIN","FIT":"FIT","FMX":"FMX","FMY":"FMY","FNA":"FNA","FND":"FND","FN":"GWM","FNQ":"NGQ","FNS":"NGS","FNY":"FNY","FOX":"FOX","FPD":"FPD","FPH":"FPH","FRC":"FRC","FSO":"SOY","FTH":"FTH","FVN":"FVN","FVS":"FVS","GAS":"G","GB3":"GB3","GBH":"GBH","GBO":"GBO","GBP":"GBP","GBQ":"GBP","GBR":"GBR","GBW":"GBW","GDI":"GDI","GDS":"GDS","GDV":"GDV","GGI":"GGI","GLD":"GLD","GOL":"GOL","GT":"GT","GWT":"GWT","GWY":"GWY","GX":"GX","HOS":"HOS","HOU":"HOU","HSI":"HSI","HT":"HT","H":"H","HWF":"HWF","I":"I02","IB":"IB","ICF":"ICF","IE":"I","IET":"IT","IHO":"UHO","IND":"IND","IRB":"UHU","IRBT":"LRT","IR":"IR","ISE":"ISE","ISP":"ISP","JAP":"JAP","JG":"JG","JPI":"JPY","JRT":"JRT","KAN":"KAN","KLI":"KLI","KPO":"CPO","LCE":"LCE","LCP":"LCP","LF":"Z","LG":"R","LGT":"RT","LHT":"LHT","LHW":"USW","LIV":"USO","LJN":"USP","LLR":"FEF","LLS":"FEO","LRC":"RC","LRCT":"RCT","LU":"TWS","LY":"Y2","M":"M70","MAE":"MAE","MAS":"MAS","MAW":"MAW","MAX":"MFA","MBE":"MBE","MBZ":"MBZ","MC2":"MCP","MCH":"MCH","MDK":"MDK","MDM":"MDM","MEA":"MEA","MEE":"MEE","MEF":"MEF","MEL":"MEL","MEM":"MEM","MEN":"MEN","MEP":"MEP","MEX":"MEX","MFI":"MFI","MFR":"MFR","MGA":"MGA","MGC":"MGC","MGS":"MGS","MJP":"MJP","MKU":"MKU","MMY":"MMY","MNL":"MNL","MNO":"MNA","MNW":"MNW","MRM":"MRM","MRQ":"MRQ","MRW":"MRW","MSA":"MSA","MSD":"MSD","MSI":"MSI","MSJ":"MSJ","MSP":"MSP","MSR":"MSM","MSS":"MSS","MST":"MST","MSW":"MSW","MSZ":"MSZ","MT":"FCE","MTH":"MTH","MTW":"MTW","MWA":"WM","MWB":"MWB","MWC":"MWC","MWD":"MWD","MWF":"MWF","MWH":"MWH","MWI":"MWI","MWO":"MWO","MWP":"MWP","MWQ":"MWQ","MWR":"MWR","MWT":"MWT","MXG":"MXG","MXN":"MXN","MZA":"MZA","NAU":"NAU","NCQ":"NCQ","NCY":"NCY","NDI":"NDI","NEA":"NEA","NID":"NID","NN":"NK","NPH":"NPH","NTH":"NTH","NTW":"NTW","NU":"NU","NXJ":"NXJ","NZD":"NZD","NZL":"NZL","OAM":"OAM","OAT":"OAT","OE":"O","OFR1":"SF1","OFR3":"SF3","OIL":"OIL","OP":"P","PG":"ECO","PLA":"PLA","PM":"EBM","PNS":"PN","POL":"POL","PQ":"PQ","PSI":"PSI","PVF":"PVF","PV":"PV","PWF":"PWF","QC":"C","QCT":"LCT","QK":"T","QUME":"UME","QW":"W","QWT":"WT","RED":"RED","RES":"RES","SA3":"SA3","SA":"ESA","SBD":"SBD","SCE":"SCE","SCI":"SCI","SCP":"SCP","SDX":"SDX","SED":"SED","SEE":"STE","SEG":"SEG","SID":"SID","SIN":"ESI","SJC":"SJC","SJ":"EST","SK":"ESH","SLI":"SLI","SLS":"SLS","SLV":"SLV","SMD":"SMD","SMM":"SMM","SMS":"SMS","SMX":"SMX","SND":"ND","SNS":"NS","SO3":"SO3","SOA":"SOA","SPI":"AP","SQ":"STX","SRI":"SRI","SSE":"SSE","SS":"SGP","SSX":"SSX","ST3":"ST3","STD":"STD","STF":"STF","STG":"STG","STI":"ST","STJ":"STJ","STL":"STL","STM":"STM","STN":"STI","STO":"STO","STP":"STP","STQ":"STQ","STS":"STS","STU":"STU","STW":"STW","SU1":"SU1","SU2":"SU2","SU3":"SU3","SU5":"SU5","SUD":"SUD","SUN":"SUN","SUS":"SUS","SWF":"SWF","SW":"SMI","SWI":"SWI","SXE":"SXE","SX":"ESB","SY":"ESY","TA":"STB","TC":"STC","TDD":"TDD","TD":"TDX","TF":"STA","TIN":"TIN","TJ":"STH","TL":"STV","TM":"STN","TP":"STZ","TR":"STR","TT":"STT","TUK":"TUK","TWN":"TWN","TX":"STY","UAA":"UAA","UAL":"UAL","UAM":"UAM","UAQ":"UAQ","UAS":"UAS","UAV":"UAV","UBL":"UBL","UBQ":"UBQ","U":"U","UKA":"UKA","UKD":"UKD","UNF":"UNF","UPL":"UPL","UPO":"UPO","US3":"US3","UT":"UT","UZQ":"UZQ","WBT":"WBT","WDO":"WDO","WEA":"WEA","WEU":"WEU","WIN":"WIN","WMA":"WMA","WPD":"WPD","WPT":"WPT","WTO":"WTO","XFC":"XFC","XG":"DAX","XGL":"XGL","XSF":"XSF","XT":"XT","XXE":"XXE","XXP":"XXP","XXS":"XXS","YMA":"YMA","YPT":"YPT","YT":"YT","ZAR":"ZAR"} \ No newline at end of file diff --git a/QuantConnect.IQFeed/IQFeedAPI/IQLookupHistorySymbolClient.cs b/QuantConnect.IQFeed/IQFeedAPI/IQLookupHistorySymbolClient.cs index 5535d8e..eae9a08 100644 --- a/QuantConnect.IQFeed/IQFeedAPI/IQLookupHistorySymbolClient.cs +++ b/QuantConnect.IQFeed/IQFeedAPI/IQLookupHistorySymbolClient.cs @@ -23,8 +23,14 @@ namespace QuantConnect.Lean.DataSource.IQFeed // Historical stock data lookup events public class LookupTickEventArgs : LookupEventArgs { - public LookupTickEventArgs(string requestId, string line) : - base(requestId, LookupType.REQ_HST_TCK, LookupSequence.MessageDetail) + public DateTime DateTimeStamp { get; } + public decimal Last { get; } + public decimal LastSize { get; } + public decimal Bid { get; } + public decimal Ask { get; } + + public LookupTickEventArgs(string requestId, string line) + : base(requestId, LookupType.REQ_HST_TCK, LookupSequence.MessageDetail) { var fields = line.Split(','); if (fields.Length < 11) @@ -32,41 +38,28 @@ public LookupTickEventArgs(string requestId, string line) : Log.Error("LookupIntervalEventArgs.ctor(): " + line); return; } - if (!DateTime.TryParseExact(fields[1], "yyyy-MM-dd HH:mm:ss", _enUS, DateTimeStyles.None, out _dateTimeStamp)) _dateTimeStamp = DateTime.MinValue; - if (!double.TryParse(fields[2], out _last)) _last = 0; - if (!int.TryParse(fields[3], out _lastSize)) _lastSize = 0; - if (!int.TryParse(fields[4], out _totalVolume)) _totalVolume = 0; - if (!double.TryParse(fields[5], out _bid)) _bid = 0; - if (!double.TryParse(fields[6], out _ask)) _ask = 0; - if (!int.TryParse(fields[7], out _tickId)) _tickId = 0; - if (!char.TryParse(fields[10], out _basis)) _basis = ' '; - } - public DateTime DateTimeStamp { get { return _dateTimeStamp; } } - public double Last { get { return _last; } } - public int LastSize { get { return _lastSize; } } - public int TotalVolume { get { return _totalVolume; } } - public double Bid { get { return _bid; } } - public double Ask { get { return _ask; } } - public int TickId { get { return _tickId; } } - public char Basis { get { return _basis; } } - #region private - private DateTime _dateTimeStamp; - private double _last; - private int _lastSize; - private int _totalVolume; - private double _bid; - private double _ask; - private int _tickId; - private char _basis; - private CultureInfo _enUS = new CultureInfo("en-US"); - #endregion + DateTimeStamp = DateTime.TryParseExact(fields[1], "yyyy-MM-dd HH:mm:ss", new CultureInfo("en-US"), DateTimeStyles.None, out var dateTimeStamp) ? dateTimeStamp : default; + + Last = decimal.TryParse(fields[2], out var l) ? l : 0m; + LastSize = decimal.TryParse(fields[3], out var ls) ? ls : 0m; + Bid = decimal.TryParse(fields[5], out var b) ? b : 0m; + Ask = decimal.TryParse(fields[6], out var a) ? a : 0m; + } } public class LookupIntervalEventArgs : LookupEventArgs { - public LookupIntervalEventArgs(string requestId, string line) : - base(requestId, LookupType.REQ_HST_INT, LookupSequence.MessageDetail) + public DateTime DateTimeStamp { get; } + public decimal High { get; } + public decimal Low { get; } + public decimal Open { get; } + public decimal Close { get; } + public int TotalVolume { get; } + public int PeriodVolume { get; } + + public LookupIntervalEventArgs(string requestId, string line) + : base(requestId, LookupType.REQ_HST_INT, LookupSequence.MessageDetail) { var fields = line.Split(','); if (fields.Length < 8) @@ -74,38 +67,29 @@ public LookupIntervalEventArgs(string requestId, string line) : Log.Error("LookupIntervalEventArgs.ctor(): " + line); return; } - if (!DateTime.TryParseExact(fields[1], "yyyy-MM-dd HH:mm:ss", _enUS, DateTimeStyles.None, out _dateTimeStamp)) _dateTimeStamp = DateTime.MinValue; - if (!double.TryParse(fields[2], out _high)) _high = 0; - if (!double.TryParse(fields[3], out _low)) _low = 0; - if (!double.TryParse(fields[4], out _open)) _open = 0; - if (!double.TryParse(fields[5], out _close)) _close = 0; - if (!int.TryParse(fields[6], out _totalVolume)) _totalVolume = 0; - if (!int.TryParse(fields[7], out _periodVolume)) _periodVolume = 0; - } - public DateTime DateTimeStamp { get { return _dateTimeStamp; } } - public double High { get { return _high; } } - public double Low { get { return _low; } } - public double Open { get { return _open; } } - public double Close { get { return _close; } } - public int TotalVolume { get { return _totalVolume; } } - public int PeriodVolume { get { return _periodVolume; } } - #region private - private DateTime _dateTimeStamp; - private double _high; - private double _low; - private double _open; - private double _close; - private int _totalVolume; - private int _periodVolume; - private CultureInfo _enUS = new CultureInfo("en-US"); - #endregion + DateTimeStamp = DateTime.TryParseExact(fields[1], "yyyy-MM-dd HH:mm:ss", new CultureInfo("en-US"), DateTimeStyles.None, out var dateTimeStamp) ? dateTimeStamp : default; + High = decimal.TryParse(fields[2], out var h) ? h : 0m; + Low = decimal.TryParse(fields[3], out var l) ? l : 0m; + Open = decimal.TryParse(fields[4], out var o) ? o : 0m; + Close = decimal.TryParse(fields[5], out var c) ? c : 0m; + TotalVolume = int.TryParse(fields[6], out var t) ? t : 0; + PeriodVolume = int.TryParse(fields[7], out var p) ? p : 0; + } } public class LookupDayWeekMonthEventArgs : LookupEventArgs { - public LookupDayWeekMonthEventArgs(string requestId, string line) : - base(requestId, LookupType.REQ_HST_DWM, LookupSequence.MessageDetail) + public DateTime DateTimeStamp { get; } + public decimal High { get; } + public decimal Low { get; } + public decimal Open { get; } + public decimal Close { get; } + public int PeriodVolume { get; } + public int OpenInterest { get; } + + public LookupDayWeekMonthEventArgs(string requestId, string line) + : base(requestId, LookupType.REQ_HST_DWM, LookupSequence.MessageDetail) { var fields = line.Split(','); if (fields.Length < 8) @@ -113,32 +97,19 @@ public LookupDayWeekMonthEventArgs(string requestId, string line) : Log.Error("LookupIntervalEventArgs.ctor(): " + line); return; } - if (!DateTime.TryParseExact(fields[1], "yyyy-MM-dd HH:mm:ss", _enUS, DateTimeStyles.None, out _dateTimeStamp)) _dateTimeStamp = DateTime.MinValue; - if (!double.TryParse(fields[2], out _high)) _high = 0; - if (!double.TryParse(fields[3], out _low)) _low = 0; - if (!double.TryParse(fields[4], out _open)) _open = 0; - if (!double.TryParse(fields[5], out _close)) _close = 0; - if (!int.TryParse(fields[6], out _periodVolume)) _periodVolume = 0; - if (!int.TryParse(fields[7], out _openInterest)) _openInterest = 0; + + + DateTimeStamp = DateTime.TryParseExact(fields[1], DateFormat.DB, new CultureInfo("en-US"), DateTimeStyles.None, out var dateTimeStamp) ? dateTimeStamp : default; + High = decimal.TryParse(fields[2], out var h) ? h : 0m; + Low = decimal.TryParse(fields[3], out var l) ? l : 0m; + Open = decimal.TryParse(fields[4], out var o) ? o : 0m; + Close = decimal.TryParse(fields[5], out var c) ? c : 0m; + PeriodVolume = int.TryParse(fields[6], out var p) ? p : 0; + OpenInterest = int.TryParse(fields[7], out var t) ? t : 0; } - public DateTime DateTimeStamp { get { return _dateTimeStamp; } } - public double High { get { return _high; } } - public double Low { get { return _low; } } - public double Open { get { return _open; } } - public double Close { get { return _close; } } - public int PeriodVolume { get { return _periodVolume; } } - public int OpenInterest { get { return _openInterest; } } - #region private - private DateTime _dateTimeStamp; - private double _high; - private double _low; - private double _open; - private double _close; - private int _periodVolume; - private int _openInterest; - private CultureInfo _enUS = new CultureInfo("en-US"); - #endregion + + } // Symbol search lookup events diff --git a/QuantConnect.IQFeed/IQFeedDataDownloader.cs b/QuantConnect.IQFeed/IQFeedDataDownloader.cs index 0bcb5d0..fed6e07 100644 --- a/QuantConnect.IQFeed/IQFeedDataDownloader.cs +++ b/QuantConnect.IQFeed/IQFeedDataDownloader.cs @@ -14,12 +14,13 @@ * */ +using NodaTime; using QuantConnect.Data; -using QuantConnect.Logging; using QuantConnect.Securities; using QuantConnect.Data.Market; using QuantConnect.Configuration; using IQFeed.CSharpApiClient.Lookup; +using System.Collections.Concurrent; namespace QuantConnect.Lean.DataSource.IQFeed { @@ -33,6 +34,16 @@ public class IQFeedDataDownloader : IDataDownloader /// private const int NumberOfClients = 8; + /// + /// Provides access to all available symbols corresponding to a canonical symbol using the IQFeed data source. + /// + private readonly IQFeedDataQueueUniverseProvider _dataQueueUniverseProvider; + + /// + /// Provides access to exchange hours and raw data times zones in various markets + /// + private readonly MarketHoursDatabase _marketHoursDatabase; + /// /// Lazy initialization for the IQFeed file history provider. /// @@ -51,6 +62,9 @@ public class IQFeedDataDownloader : IDataDownloader /// public IQFeedDataDownloader() { + _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + _dataQueueUniverseProvider = new IQFeedDataQueueUniverseProvider(); + _fileHistoryProviderLazy = new Lazy(() => { // Create and connect the IQFeed lookup client @@ -58,7 +72,7 @@ public IQFeedDataDownloader() // Establish connection with IQFeed Client lookupClient.Connect(); - return new IQFeedFileHistoryProvider(lookupClient, new IQFeedDataQueueUniverseProvider(), MarketHoursDatabase.FromDataFolder()); + return new IQFeedFileHistoryProvider(lookupClient, _dataQueueUniverseProvider, MarketHoursDatabase.FromDataFolder()); }); } @@ -75,22 +89,120 @@ public IQFeedDataDownloader() var endUtc = dataDownloaderGetParameters.EndUtc; var tickType = dataDownloaderGetParameters.TickType; + var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); + var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); + var dataType = resolution == Resolution.Tick ? typeof(Tick) : typeof(TradeBar); - return _fileHistoryProvider.ProcessHistoryRequests( - new HistoryRequest( + if (symbol.IsCanonical()) + { + return GetCanonicalSymbolHistory( + symbol, startUtc, endUtc, dataType, - symbol, - resolution, - SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), - TimeZones.NewYork, resolution, - true, - false, - DataNormalizationMode.Adjusted, - tickType)); + exchangeHours, + dataTimeZone, + dataDownloaderGetParameters.TickType); + } + else + { + return _fileHistoryProvider.ProcessHistoryRequests( + new HistoryRequest( + startUtc, + endUtc, + dataType, + symbol, + resolution, + exchangeHours, + dataTimeZone, + resolution, + true, + false, + DataNormalizationMode.Adjusted, + tickType)); + } + } + + /// + /// Retrieves historical data for all individual tradeable contracts derived from a given canonical symbol + /// (such as options, futures, or other security types) within the specified date range, resolution, and tick type. + /// + /// The canonical representing the underlying security group (option chain, future chain, etc.). + /// The UTC start time of the historical data request. + /// The UTC end time of the historical data request. + /// The type of data to retrieve, such as , , or . + /// The resolution of the historical data (e.g., Minute, Hour, Daily). + /// The exchange hours for the underlying security to correctly filter trading times. + /// The time zone for the timestamps in the returned data. + /// The tick type to retrieve, such as Trade or Quote ticks. + /// + /// An enumerable collection of instances representing historical data for all tradeable contracts + /// derived from the canonical symbol within the requested date range. Returns null if no data was found. + /// + private IEnumerable? GetCanonicalSymbolHistory(Symbol symbol, DateTime startUtc, DateTime endUtc, Type dataType, + Resolution resolution, SecurityExchangeHours exchangeHours, DateTimeZone dataTimeZone, TickType tickType) + { + var blockingCollection = new BlockingCollection(); + var symbols = GetCanonicalSymbolChain(symbol, startUtc, endUtc); + + // Symbol can have a lot of Option parameters + Task.Run(() => Parallel.ForEach(symbols, targetSymbol => + { + var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, targetSymbol, resolution, exchangeHours, dataTimeZone, + resolution, true, false, DataNormalizationMode.Raw, tickType); + + var history = _fileHistoryProvider.ProcessHistoryRequests(historyRequest); + + // If history is null, it indicates an incorrect or missing request for historical data, + // so we skip processing for this symbol and move to the next one. + if (history == null) + { + return; + } + + foreach (var data in history) + { + blockingCollection.Add(data); + } + })).ContinueWith(_ => + { + blockingCollection.CompleteAdding(); + }); + + var historyResponses = blockingCollection.GetConsumingEnumerable(); + + // Validate if the collection contains at least one successful response from history. + if (!historyResponses.Any()) + { + return null; + } + + return historyResponses; + } + + /// + /// Retrieves a distinct set of tradeable option symbols for the specified underlying security + /// within the given date range, filtered to trading days as defined by the market hours database. + /// This includes all option types such as equity options, futures options, index options, and more, + /// that are available for trading on those dates. + /// + /// The canonical symbol representing the underlying security (e.g., equity, future, or index). + /// The UTC start date of the lookup period. + /// The UTC end date of the lookup period. + /// + /// An enumerable collection of unique instances representing tradeable options + /// for the specified underlying symbol over the specified date range. + /// + protected virtual IEnumerable GetCanonicalSymbolChain(Symbol symbol, DateTime startUtc, DateTime endUtc) + { + var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); + + return QuantConnect.Time.EachTradeableDay(exchangeHours, startUtc.Date, endUtc.Date) + .Select(date => _dataQueueUniverseProvider.LookupSymbols(symbol, default, default)) + .SelectMany(x => x) + .Distinct(); } } } diff --git a/QuantConnect.IQFeed/IQFeedDataProvider.cs b/QuantConnect.IQFeed/IQFeedDataProvider.cs index 417601a..a023823 100644 --- a/QuantConnect.IQFeed/IQFeedDataProvider.cs +++ b/QuantConnect.IQFeed/IQFeedDataProvider.cs @@ -22,17 +22,18 @@ using QuantConnect.Api; using QuantConnect.Util; using QuantConnect.Data; -using System.Diagnostics; using Newtonsoft.Json.Linq; using QuantConnect.Packets; using QuantConnect.Logging; +using QuantConnect.Securities; using QuantConnect.Interfaces; using QuantConnect.Data.Market; using QuantConnect.Configuration; -using Timer = System.Timers.Timer; using System.Security.Cryptography; using System.Collections.Concurrent; using System.Net.NetworkInformation; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Lean.Engine.HistoricalData; using HistoryRequest = QuantConnect.Data.HistoryRequest; namespace QuantConnect.Lean.DataSource.IQFeed @@ -40,7 +41,7 @@ namespace QuantConnect.Lean.DataSource.IQFeed /// /// IQFeedDataProvider is an implementation of IDataQueueHandler and IHistoryProvider /// - public class IQFeedDataProvider : HistoryProviderBase, IDataQueueHandler, IDataQueueUniverseProvider + public class IQFeedDataProvider : SynchronizingHistoryProvider, IDataQueueHandler, IDataQueueUniverseProvider { private bool _isConnected; private readonly HashSet _symbols; @@ -48,6 +49,11 @@ public class IQFeedDataProvider : HistoryProviderBase, IDataQueueHandler, IDataQ private readonly object _sync = new object(); private IQFeedDataQueueUniverseProvider _symbolUniverse; + /// + /// Represents the time zone used by IQFeed, which returns time in the New York (EST) Time Zone with daylight savings time. + /// + public readonly static DateTimeZone TimeZoneIQFeed = TimeZones.NewYork; + //Socket connections: private AdminPort _adminPort; private Level1Port _level1Port; @@ -230,17 +236,18 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) /// An enumerable of the slices of data covering the span specified in each request public override IEnumerable? GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) { - var subscriptions = new List>(); + var subscriptions = new List(); foreach (var request in requests) { - var history = _historyPort.ProcessHistoryRequests(request); + var history = _historyPort.ProcessHistoryRequests(request, sliceTimeZone); if (history == null) { continue; } - subscriptions.Add(history); + var subscription = CreateSubscription(request, history); + subscriptions.Add(subscription); } if (subscriptions.Count == 0) @@ -248,7 +255,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return null; } - return subscriptions.SelectMany(x => x); + return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); } /// @@ -556,66 +563,53 @@ public AdminPort() /// public class Level1Port : IQLevel1Client { - private int count; - private DateTime start; - private DateTime _feedTime; - private Stopwatch _stopwatch = new Stopwatch(); - private readonly Timer _timer; private readonly ConcurrentDictionary _prices; private readonly ConcurrentDictionary _openInterests; private readonly IQFeedDataQueueUniverseProvider _symbolUniverse; private readonly IDataAggregator _aggregator; - private int _dataQueueCount; - public DateTime FeedTime - { - get - { - if (_feedTime == new DateTime()) return DateTime.Now; - return _feedTime.AddMilliseconds(_stopwatch.ElapsedMilliseconds); - } - set - { - _feedTime = value; - _stopwatch = Stopwatch.StartNew(); - } - } + /// + /// A thread-safe dictionary that maps a to a . + /// + /// + /// This dictionary is used to store the time zone information for each symbol in a concurrent environment, + /// ensuring thread safety when accessing or modifying the time zone data. + /// + private readonly ConcurrentDictionary _exchangeTimeZoneByLeanSymbol = new(); public Level1Port(IDataAggregator aggregator, IQFeedDataQueueUniverseProvider symbolUniverse) : base(IQFeedDefault.BufferSize) { - start = DateTime.Now; _prices = new ConcurrentDictionary(); _openInterests = new ConcurrentDictionary(); _aggregator = aggregator; _symbolUniverse = symbolUniverse; Level1SummaryUpdateEvent += OnLevel1SummaryUpdateEvent; - Level1TimerEvent += OnLevel1TimerEvent; Level1ServerDisconnectedEvent += OnLevel1ServerDisconnected; Level1ServerReconnectFailed += OnLevel1ServerReconnectFailed; Level1UnknownEvent += OnLevel1UnknownEvent; Level1FundamentalEvent += OnLevel1FundamentalEvent; + } - _timer = new Timer(1000); - _timer.Enabled = false; - _timer.AutoReset = true; - _timer.Elapsed += (sender, args) => + /// + /// Returns a timestamp for a tick converted to the exchange time zone + /// + private DateTime GetRealTimeTickTime(Symbol symbol) + { + var time = DateTime.UtcNow; + var exchangeTimeZone = default(DateTimeZone); + lock (_exchangeTimeZoneByLeanSymbol) { - var ticksPerSecond = count / (DateTime.Now - start).TotalSeconds; - int dataQueueCount = Interlocked.Exchange(ref _dataQueueCount, 0); - if (ticksPerSecond > 1000 || dataQueueCount > 31) + if (!_exchangeTimeZoneByLeanSymbol.TryGetValue(symbol, out exchangeTimeZone)) { - Log.Trace($"IQFeed.OnSecond(): Ticks/sec: {ticksPerSecond.ToStringInvariant("0000.00")} " + - $"Engine.Ticks.Count: {dataQueueCount} CPU%: {OS.CpuUsage.ToStringInvariant("0.0") + "%"}" - ); + // read the exchange time zone from market-hours-database + exchangeTimeZone = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; + _exchangeTimeZoneByLeanSymbol[symbol] = exchangeTimeZone; } + } - count = 0; - start = DateTime.Now; - }; - - _timer.Enabled = true; + return time.ConvertFromUtc(exchangeTimeZone); } private Symbol GetLeanSymbol(string ticker) @@ -635,7 +629,7 @@ private void OnLevel1FundamentalEvent(object sender, Level1FundamentalEventArgs _prices.TryGetValue(e.Symbol, out referencePrice); var symbol = GetLeanSymbol(e.Symbol); - var split = new Data.Market.Split(symbol, FeedTime, (decimal)referencePrice, (decimal)e.SplitFactor1, SplitType.SplitOccurred); + var split = new Data.Market.Split(symbol, GetRealTimeTickTime(symbol), (decimal)referencePrice, (decimal)e.SplitFactor1, SplitType.SplitOccurred); Emit(split); } } @@ -657,29 +651,20 @@ private void OnLevel1SummaryUpdateEvent(object sender, Level1SummaryUpdateEventA && e.TypeOfUpdate != Level1SummaryUpdateEventArgs.UpdateType.Bid && e.TypeOfUpdate != Level1SummaryUpdateEventArgs.UpdateType.Ask) return; - count++; - var time = FeedTime; var last = (decimal)(e.TypeOfUpdate == Level1SummaryUpdateEventArgs.UpdateType.ExtendedTrade ? e.ExtendedTradingLast : e.Last); var symbol = GetLeanSymbol(e.Symbol); + var time = GetRealTimeTickTime(symbol); TickType tradeType; switch (symbol.ID.SecurityType) { - // the feed time is in NYC/EDT, convert it into EST case SecurityType.Forex: - - time = FeedTime.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard); // TypeOfUpdate always equal to UpdateType.Trade for FXCM, but the message contains B/A and last data tradeType = TickType.Quote; - break; - - // for all other asset classes we leave it as is (NYC/EDT) default: - - time = FeedTime; tradeType = e.TypeOfUpdate == Level1SummaryUpdateEventArgs.UpdateType.Bid || e.TypeOfUpdate == Level1SummaryUpdateEventArgs.UpdateType.Ask ? TickType.Quote : @@ -687,7 +672,7 @@ private void OnLevel1SummaryUpdateEvent(object sender, Level1SummaryUpdateEventA break; } - var tick = new Tick(time, symbol, last, (decimal)e.Bid, (decimal)e.Ask) + var tick = new Tick(GetRealTimeTickTime(symbol), symbol, last, (decimal)e.Bid, (decimal)e.Ask) { AskSize = e.AskSize, BidSize = e.BidSize, @@ -713,19 +698,6 @@ private void OnLevel1SummaryUpdateEvent(object sender, Level1SummaryUpdateEventA private void Emit(BaseData tick) { _aggregator.Update(tick); - Interlocked.Increment(ref _dataQueueCount); - } - - /// - /// Set the interal clock time. - /// - private void OnLevel1TimerEvent(object sender, Level1TimerEventArgs e) - { - //If there was a bad tick and the time didn't set right, skip setting it here and just use our millisecond timer to set the time from last time it was set. - if (e.DateTimeStamp != DateTime.MinValue) - { - FeedTime = e.DateTimeStamp; - } } /// @@ -756,8 +728,8 @@ private void OnLevel1UnknownEvent(object sender, Level1TextLineEventArgs e) // this type is expected to be used for exactly one job at a time public class HistoryPort : IQLookupHistorySymbolClient { - private bool _inProgress; - private ConcurrentDictionary _requestDataByRequestId; + private AutoResetEvent _historyRequestResetEvent = new(false); + private readonly ConcurrentDictionary _requestDataByRequestId = []; private ConcurrentDictionary> _currentRequest; private readonly IQFeedDataQueueUniverseProvider _symbolUniverse; @@ -783,7 +755,6 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse) : base(IQFeedDefault.BufferSize) { _symbolUniverse = symbolUniverse; - _requestDataByRequestId = new ConcurrentDictionary(); _currentRequest = new ConcurrentDictionary>(); } @@ -798,9 +769,14 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo } /// - /// Populate request data + /// Processes the specified historical data request and generates a sequence of instances. /// - public IEnumerable? ProcessHistoryRequests(HistoryRequest request) + /// The historical data requests + /// The time zone used when time stamping the slice instances + /// An enumerable sequence of objects representing the historical data for the request, + /// or null if no data could be retrieved or processed. + /// + public IEnumerable? ProcessHistoryRequests(HistoryRequest request, DateTimeZone sliceTimeZone) { // skipping universe and canonical symbols if (!CanHandle(request.Symbol) || @@ -835,13 +811,9 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo return null; } - // Set this process status - _inProgress = true; - var ticker = _symbolUniverse.GetBrokerageSymbol(request.Symbol); - var start = request.StartTimeUtc.ConvertFromUtc(TimeZones.NewYork); - DateTime? end = request.EndTimeUtc.ConvertFromUtc(TimeZones.NewYork); - var exchangeTz = request.ExchangeHours.TimeZone; + var start = request.StartTimeUtc.ConvertFromUtc(IQFeedDataProvider.TimeZoneIQFeed); + DateTime? end = request.EndTimeUtc.ConvertFromUtc(IQFeedDataProvider.TimeZoneIQFeed); // if we're within a minute of now, don't set the end time if (request.EndTimeUtc >= DateTime.UtcNow.AddMinutes(-1)) { @@ -874,15 +846,16 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo _requestDataByRequestId[reqid] = request; - while (_inProgress) + if (!_historyRequestResetEvent.WaitOne(TimeSpan.FromSeconds(20))) { - continue; + Log.Trace($"{nameof(IQFeedDataProvider)}.{nameof(ProcessHistoryRequests)}: Timeout waiting for history data. Request details - Symbol: {request.Symbol.Value} ({request.Symbol.SecurityType}), Resolution: {request.Resolution}, StartTimeUtc: {request.StartTimeUtc:u}, EndTimeUtc: {request.EndTimeUtc:u}"); + return null; } - return GetSlice(exchangeTz); + return GetSlice(request.ExchangeHours.TimeZone, sliceTimeZone); } - private IEnumerable? GetSlice(DateTimeZone exchangeTz) + private IEnumerable? GetSlice(DateTimeZone exchangeTz, DateTimeZone sliceTimeZone) { // After all data arrive, we pass it to the algorithm through memory and write to a file foreach (var key in _currentRequest.Keys) @@ -892,7 +865,7 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo foreach (var tradeBar in tradeBars) { // Returns IEnumerable object - yield return new Slice(tradeBar.EndTime, new[] { tradeBar }, tradeBar.EndTime.ConvertToUtc(exchangeTz)); + yield return tradeBar; } } } @@ -939,14 +912,13 @@ protected override void OnLookupEvent(LookupEventArgs e) _currentRequest.AddOrUpdate(e.Id, new List()); break; case LookupSequence.MessageDetail: - List current; - if (_currentRequest.TryGetValue(e.Id, out current)) + if (_currentRequest.TryGetValue(e.Id, out var current)) { HandleMessageDetail(e, current); } break; case LookupSequence.MessageEnd: - _inProgress = false; + _historyRequestResetEvent.Set(); break; default: throw new ArgumentOutOfRangeException(); @@ -965,8 +937,7 @@ protected override void OnLookupEvent(LookupEventArgs e) /// Current list of BaseData object private void HandleMessageDetail(LookupEventArgs e, List current) { - var requestData = _requestDataByRequestId[e.Id]; - var data = GetData(e, requestData); + var data = GetData(e, _requestDataByRequestId[e.Id]); if (data != null && data.Time != DateTime.MinValue) { current.Add(data); @@ -981,54 +952,53 @@ private void HandleMessageDetail(LookupEventArgs e, List current) /// BaseData object private BaseData GetData(LookupEventArgs e, HistoryRequest requestData) { - var isEquity = requestData.Symbol.SecurityType == SecurityType.Equity; try { switch (e.Type) { case LookupType.REQ_HST_TCK: - var t = (LookupTickEventArgs)e; - var time = isEquity ? t.DateTimeStamp : t.DateTimeStamp.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard); - switch (requestData.TickType) + if (e is LookupTickEventArgs t) { - case TickType.Trade: - return new Tick() - { - Time = time, - Value = (decimal)t.Last, - DataType = MarketDataType.Tick, - Symbol = requestData.Symbol, - TickType = TickType.Trade, - Quantity = t.LastSize, - }; - case TickType.Quote: - return new Tick() - { - Time = time, - DataType = MarketDataType.Tick, - Symbol = requestData.Symbol, - TickType = TickType.Quote, - AskPrice = (decimal)t.Ask, - //AskSize = askSize, - BidPrice = (decimal)t.Bid, - //BidSize = bidSize, - }; - default: - throw new NotImplementedException($"The TickType '{requestData.TickType}' is not supported in the {nameof(GetData)} method. Please implement the necessary logic for handling this TickType."); + var tick = new Tick() + { + Time = t.DateTimeStamp.ConvertTo(IQFeedDataProvider.TimeZoneIQFeed, requestData.ExchangeHours.TimeZone), + DataType = MarketDataType.Tick, + Symbol = requestData.Symbol, + }; + + switch (requestData.TickType) + { + case TickType.Trade: + tick.TickType = TickType.Trade; + tick.Value = t.Last; + tick.Quantity = t.LastSize; + break; + case TickType.Quote: + tick.TickType = TickType.Quote; + tick.AskPrice = t.Ask; + tick.BidPrice = t.Bid; + break; + default: + throw new NotImplementedException($"The TickType '{requestData.TickType}' is not supported in the {nameof(GetData)} method. Please implement the necessary logic for handling this TickType."); + } + + return tick; } + return null; case LookupType.REQ_HST_INT: - var i = (LookupIntervalEventArgs)e; - if (i.DateTimeStamp == DateTime.MinValue) return null; - var istartTime = i.DateTimeStamp - requestData.Resolution.ToTimeSpan(); - if (!isEquity) istartTime = istartTime.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard); - return new TradeBar(istartTime, requestData.Symbol, (decimal)i.Open, (decimal)i.High, (decimal)i.Low, (decimal)i.Close, i.PeriodVolume, requestData.Resolution.ToTimeSpan()); + if (e is LookupIntervalEventArgs i && i.DateTimeStamp != default) + { + var resolutionTimeSpan = requestData.Resolution.ToTimeSpan(); + var time = (i.DateTimeStamp - resolutionTimeSpan).ConvertTo(IQFeedDataProvider.TimeZoneIQFeed, requestData.ExchangeHours.TimeZone); + return new TradeBar(time, requestData.Symbol, i.Open, i.High, i.Low, i.Close, i.PeriodVolume, resolutionTimeSpan); + } + return null; case LookupType.REQ_HST_DWM: - var d = (LookupDayWeekMonthEventArgs)e; - if (d.DateTimeStamp == DateTime.MinValue) return null; - var dstartTime = d.DateTimeStamp.Date; - if (!isEquity) dstartTime = dstartTime.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard); - return new TradeBar(dstartTime, requestData.Symbol, (decimal)d.Open, (decimal)d.High, (decimal)d.Low, (decimal)d.Close, d.PeriodVolume, requestData.Resolution.ToTimeSpan()); - + if (e is LookupDayWeekMonthEventArgs d && d.DateTimeStamp != default) + { + return new TradeBar(d.DateTimeStamp.Date.ConvertTo(IQFeedDataProvider.TimeZoneIQFeed, requestData.ExchangeHours.TimeZone), requestData.Symbol, d.Open, d.High, d.Low, d.Close, d.PeriodVolume, requestData.Resolution.ToTimeSpan()); + } + return null; // we don't need to handle these other types case LookupType.REQ_SYM_SYM: case LookupType.REQ_SYM_SIC: diff --git a/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs b/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs index c8f11b1..1dd719f 100644 --- a/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs +++ b/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs @@ -22,9 +22,12 @@ using QuantConnect.Interfaces; using QuantConnect.Brokerages; using QuantConnect.Securities; +using System.Runtime.CompilerServices; using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.DataFeeds.Transport; +[assembly: InternalsVisibleTo("QuantConnect.Lean.DataSource.IQFeed.Tests")] + namespace QuantConnect.Lean.DataSource.IQFeed { /// @@ -67,11 +70,26 @@ public class IQFeedDataQueueUniverseProvider : IDataQueueUniverseProvider, ISymb // Prioritized list of exchanges used to find right futures contract public static readonly Dictionary FuturesExchanges = new Dictionary { - { "CME", Market.Globex }, + { "CBOT_GBX", Market.CBOT }, + { "CBOTMINI", Market.CBOT }, + { "CFE", Market.CFE }, + { "CME_GBX", Market.Globex }, + { "CMEMINI", Market.CME }, + { "COMEX_GBX", Market.COMEX }, + { "EUREX", Market.EUREX }, + { "ICEEA", Market.ICE }, // ICE Futures Europe - included in both Commodities/Financials packages + { "ICEEC", Market.ICE }, // ICE Futures Europe - Commodities + { "ICEEF", Market.ICE }, // ICE Futures Europe - Financials + { "ICEENDEX", Market.ICE }, // ICE Endex + { "ICEFANG", Market.ICE }, // ICE FANG Futures + { "ICEFC", Market.ICE }, // ICE Futures Canada + { "ICEFU", Market.ICE }, // ICE Futures US + { "NYMEX_GBX", Market.NYMEX }, // Nymex Globex Contracts + { "NYMEXMINI", Market.NYMEX }, // NYMEX Mini Contracts + { "SGX", Market.SGX }, // Singapore International Monetary Exchange + { "CME", Market.CME }, { "NYMEX", Market.NYMEX }, { "CBOT", Market.CBOT }, - { "ICEFU", Market.ICE }, - { "CFE", Market.CFE } }; // futures fundamental data resolver @@ -96,8 +114,7 @@ public IQFeedDataQueueUniverseProvider() /// IQFeed ticker public string GetBrokerageSymbol(Symbol symbol) { - string leanSymbol; - return _symbols.TryGetValue(symbol, out leanSymbol) ? leanSymbol : string.Empty; + return _symbols.TryGetValue(symbol, out var brokerageSymbol) ? brokerageSymbol : string.Empty; } /// @@ -269,7 +286,7 @@ private IEnumerable LoadSymbols() if (mapExists) { Log.Trace($"{nameof(IQFeedDataQueueUniverseProvider)}.{nameof(LoadSymbols)}: Loading IQFeed futures symbol map file..."); - _iqFeedNameMap = JsonConvert.DeserializeObject>(File.ReadAllText(iqfeedNameMapFullName)); + _iqFeedNameMap = JsonConvert.DeserializeObject>(File.ReadAllText(iqfeedNameMapFullName)) ?? []; } if (!universeExists) @@ -383,53 +400,76 @@ private IEnumerable LoadSymbols() case "FUTURE": - // we are not interested in designated continuous contracts - if (columns[columnSymbol].EndsWith("#") || columns[columnSymbol].EndsWith("#C") || columns[columnSymbol].EndsWith("$$")) + // Exclude non-standard contracts such as: + // - Continuous contracts (e.g., @BO# or @BO#C) + // - Synthetic or spot instruments (e.g., CVUY$$) + // These are not standard, tradable futures and are not supported by Lean + if (columns[columnSymbol].EndsWith('#') + || columns[columnSymbol].EndsWith("#C", StringComparison.InvariantCultureIgnoreCase) + || columns[columnSymbol].EndsWith("$$", StringComparison.InvariantCultureIgnoreCase)) + { continue; + } - var futuresTicker = columns[columnSymbol].TrimStart(new[] { '@' }); + var normalizedFutureBrokerageSymbol = NormalizeFuturesTicker(columns[columnListedMarket], columns[columnSymbol]); - var parsed = SymbolRepresentation.ParseFutureTicker(futuresTicker); - var underlyingString = parsed.Underlying; + var underlyingBrokerageTicker = SymbolRepresentation.ParseFutureTicker(normalizedFutureBrokerageSymbol).Underlying; - if (_iqFeedNameMap.ContainsKey(underlyingString)) - underlyingString = _iqFeedNameMap[underlyingString]; - else + if (_iqFeedNameMap.TryGetValue(underlyingBrokerageTicker, out var underlyingLeanTicker)) + { + normalizedFutureBrokerageSymbol = normalizedFutureBrokerageSymbol.Replace(underlyingBrokerageTicker, underlyingLeanTicker); + underlyingBrokerageTicker = underlyingLeanTicker; + } + else if (!mapExists && !_iqFeedNameMap.ContainsKey(underlyingBrokerageTicker)) { - if (!mapExists) + // if map is not created yet, we request this information from IQFeed + var exchangeSymbol = _symbolFundamentalData.Request(columns[columnSymbol]).Item2; + if (!string.IsNullOrEmpty(exchangeSymbol)) { - if (!_iqFeedNameMap.ContainsKey(underlyingString)) - { - // if map is not created yet, we request this information from IQFeed - var exchangeSymbol = _symbolFundamentalData.Request(columns[columnSymbol]).Item2; - if (!string.IsNullOrEmpty(exchangeSymbol)) - { - _iqFeedNameMap[underlyingString] = exchangeSymbol; - underlyingString = exchangeSymbol; - } - } + _iqFeedNameMap[underlyingBrokerageTicker] = exchangeSymbol; + underlyingBrokerageTicker = exchangeSymbol; } } - var market = GetFutureMarket(underlyingString, columns[columnExchange]); - canonicalSymbol = Symbol.Create(underlyingString, SecurityType.Future, market); + var market = GetFutureMarket(underlyingBrokerageTicker, columns[columnListedMarket]); - if (!symbolCache.ContainsKey(canonicalSymbol)) + if (TryParseFutureSymbol(normalizedFutureBrokerageSymbol, out var leanFutureSymbol)) { var placeholderSymbolData = new SymbolData { - Symbol = canonicalSymbol, + Symbol = leanFutureSymbol, SecurityCurrency = Currencies.USD, SecurityExchange = market, StartPosition = prevPosition, - EndPosition = currentPosition + EndPosition = currentPosition, + Ticker = columns[columnSymbol] }; - symbolCache.Add(canonicalSymbol, placeholderSymbolData); + if (symbolCache.TryAdd(leanFutureSymbol, placeholderSymbolData)) + { + break; + } + } + + canonicalSymbol = Symbol.Create(underlyingBrokerageTicker, SecurityType.Future, market); + + if (symbolCache.TryGetValue(canonicalSymbol, out var symbolData)) + { + symbolData.EndPosition = currentPosition; } else { - symbolCache[canonicalSymbol].EndPosition = currentPosition; + var placeholderSymbolData = new SymbolData + { + Symbol = canonicalSymbol, + SecurityCurrency = Currencies.USD, + SecurityExchange = market, + StartPosition = prevPosition, + EndPosition = currentPosition, + Ticker = columns[columnSymbol] + }; + + symbolCache.Add(canonicalSymbol, placeholderSymbolData); } break; @@ -517,7 +557,7 @@ private IEnumerable LoadSymbolOnDemand(SymbolData placeholder) continue; } - var futuresTicker = columns[columnSymbol].TrimStart(new[] { '@' }); + var futuresTicker = NormalizeFuturesTicker(columns[columnListedMarket], columns[columnSymbol]); var parsed = SymbolRepresentation.ParseFutureTicker(futuresTicker); var underlyingString = parsed.Underlying; @@ -530,7 +570,7 @@ private IEnumerable LoadSymbolOnDemand(SymbolData placeholder) continue; } - var market = GetFutureMarket(underlyingString, columns[columnExchange]); + var market = GetFutureMarket(underlyingString, columns[columnListedMarket]); // Futures contracts have different idiosyncratic expiration dates that IQFeed symbol universe file doesn't contain // We request IQFeed explicitly for the exact expiration data of each contract @@ -695,12 +735,6 @@ public Tuple Request(string ticker) } } - public IEnumerable GetBrokerageContractSymbol(Symbol subscribeSymbol) - { - return _symbols.Where(kpv => kpv.Key.SecurityType == SecurityType.Future && kpv.Key.ID.Symbol == subscribeSymbol.ID.Symbol) - .Select(kpv => kpv.Value); - } - /// /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. /// @@ -708,5 +742,44 @@ public void Dispose() { _dataCacheProvider.DisposeSafely(); } + + /// + /// Normalizes the futures ticker by removing exchange-specific electronic contract prefixes. + /// + /// The exchange code (e.g., "NYMEX", "COMEX", "CBOT", "CME"). + /// The raw symbol from the data feed. + /// The normalized symbol with the exchange-specific prefix removed. + /// + /// NYMEX, COMEX, NYMEXMINI, and NYMEX_GBX adopted the 'Q' prefix, which is removed explicitly. + /// Exchanges such as CBOT, CBOT_GBX, CBOTMINI, CME, CMEMINI, and COMEX_GBX typically use the '@' + /// prefix to denote electronic contracts. This '@' is removed by default in the fallback case. + /// + internal static string NormalizeFuturesTicker(string exchange, string symbol) + { + switch (exchange) + { + case "COMEX": + case "NYMEX": + case "NYMEXMINI": + case "NYMEX_GBX": + return symbol[1..]; + default: + return symbol[0] == '@' ? symbol[1..] : symbol; + } + } + + /// + /// Attempts to parse the specified future ticker string into a Lean . + /// + /// The future ticker string to parse. + /// + /// When this method returns, contains the parsed if the parsing succeeded; otherwise, null. + /// + /// true if the parsing succeeded; otherwise, false. + private static bool TryParseFutureSymbol(string futureTicker, out Symbol leanSymbol) + { + leanSymbol = SymbolRepresentation.ParseFutureSymbol(futureTicker); + return leanSymbol != null; + } } } diff --git a/QuantConnect.IQFeed/IQFeedFileHistoryProvider.cs b/QuantConnect.IQFeed/IQFeedFileHistoryProvider.cs index 0a91f53..06c8b8b 100644 --- a/QuantConnect.IQFeed/IQFeedFileHistoryProvider.cs +++ b/QuantConnect.IQFeed/IQFeedFileHistoryProvider.cs @@ -156,11 +156,11 @@ private IEnumerable GetDataFromFile(HistoryRequest request, string tic var tickFunc = request.TickType == TickType.Trade ? new Func(CreateTradeTick) : CreateQuoteTick; if (_filesByRequestKeyCache.TryRemove(requestKey, out filename)) - return GetDataFromTickMessages(filename, request, tickFunc, true); + return GetDataFromTickMessages(filename, request.Symbol, tickFunc, true); filename = _lookupClient.Historical.File.GetHistoryTickTimeframeAsync(ticker, startDate, endDate, dataDirection: DataDirection.Oldest).SynchronouslyAwaitTaskResult(); _filesByRequestKeyCache.AddOrUpdate(requestKey, filename); - return GetDataFromTickMessages(filename, request, tickFunc, false); + return GetDataFromTickMessages(filename, request.Symbol, tickFunc, false); case Resolution.Daily: filename = _lookupClient.Historical.File.GetHistoryDailyTimeframeAsync(ticker, startDate, endDate, dataDirection: DataDirection.Oldest).SynchronouslyAwaitTaskResult(); @@ -181,24 +181,23 @@ private IEnumerable GetDataFromFile(HistoryRequest request, string tic } /// - /// Stream IQFeed TickMessages from disk to Lean Tick + /// Streams IQFeed data from a file on disk, + /// converting each valid tick message into Lean's ticks using the provided conversion function. /// - /// - /// - /// - /// - /// Converted Tick - private IEnumerable GetDataFromTickMessages(string filename, HistoryRequest request, Func tickFunc, bool delete) + /// The path to the file containing IQFeed tick messages. + /// The symbol associated with the ticks being processed. + /// A function to convert each into a Lean object. + /// The function receives the tick timestamp, symbol, and original tick message. + /// If true, deletes the source file after processing is complete. + /// An enumerable sequence of ticks converted from the IQFeed tick messages. + private static IEnumerable GetDataFromTickMessages(string filename, Symbol symbol, Func tickFunc, bool delete) { - var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(request.Symbol.ID.Market, request.Symbol, request.Symbol.SecurityType); - // We need to discard ticks which are not impacting the price, i.e those having BasisForLast = O // To get a better understanding how IQFeed is resampling ticks, have a look to this algorithm: // https://github.com/mathpaquette/IQFeed.CSharpApiClient/blob/1b33250e057dfd6cd77e5ee35fa16aebfc8fbe79/src/IQFeed.CSharpApiClient.Extensions/Lookup/Historical/Resample/TickMessageExtensions.cs#L41 foreach (var tick in TickMessage.ParseFromFile(filename).Where(t => t.BasisForLast != 'O')) { - var timestamp = tick.Timestamp.ConvertTo(TimeZones.NewYork, dataTimeZone); - yield return tickFunc(timestamp, request.Symbol, tick); + yield return tickFunc(tick.Timestamp, symbol, tick); } if (delete)