From 5736bb421e6ead25f6799737ebfcff5d868c66da Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 28 Jul 2025 22:22:32 +0300 Subject: [PATCH 01/25] feat: update symbol-ma --- QuantConnect.IQFeed/IQFeed-symbol-map.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/QuantConnect.IQFeed/IQFeed-symbol-map.json b/QuantConnect.IQFeed/IQFeed-symbol-map.json index 40d411d..d59f843 100644 --- a/QuantConnect.IQFeed/IQFeed-symbol-map.json +++ b/QuantConnect.IQFeed/IQFeed-symbol-map.json @@ -1,2 +1,2 @@ /* This is an auto-generated file that contains mappings from IQFeed own naming to original symbols defined by respective exchanges. Delete this file if you want it to be regenerated (long operation). */ -{"1AEX":"1FT","1MT":"1FC","2AEX":"2FT","2AF":"2AF","2AW":"2AW","2EF":"2EF","2EW":"2EW","2MT":"2FC","3AF":"3AF","3AW":"3AW","4AEX":"4FT","4MT":"4FC","67T":"67T","8EF":"8EF","10Y":"10Y","20U":"20U","25U":"25U","2GT":"2GT","2VT":"2VT","2YY":"2YY","30U":"30U","30Y":"30Y","35U":"35U","3N":"Z3N","40U":"40U","45U":"45U","50U":"50U","55U":"55U","5YY":"5YY","60U":"60U","65U":"65U","6EB":"6EB","A2T":"A2T","ACD":"ACD","AC":"EH","AD":"6A","ADT":"ADT","AE":"AW","AFR":"AFR","AFT":"AFT","AJY":"AJY","ANE":"ANE","AQT":"AQT","AR":"AR","ART":"ART","AS":"AS","ASI":"ASI","ASN":"ASN","ASR":"ASR","AST":"AST","AUW":"AUW","AWN":"AWN","AWT":"AWT","B1S":"B1S","BCX":"BCX","BF":"SS","BGT":"BGT","BIB":"BIB","BIO":"BIO","BIT":"BIT","BLK":"BLK","BNB":"BNB","BOB":"BOB","BO":"ZL","BOT":"BOT","BP":"6B","BR":"6L","BSB":"BSB","BTB":"BTB","BTC":"BTC","BTE":"BTE","BT":"BOS","BUB":"BUB","C3S":"C3S","CAD":"CAD","CB":"CB","CC":"CC","CCT":"CCT","CD":"6C","C":"ZC","CH":"CHI","CHL":"CHL","CHP":"CHP","CJY":"CJY","CKO":"CZK","CKS":"CKS","CNH":"CNH","CPO":"CPO","CPV":"CPV","CSC":"CSC","CT":"CT","CTT":"CTT","CU":"CUS","CWR":"CWR","D0":"D0","D1":"D1","D1X":"D1X","D1Z":"D1Z","D2":"D2","D4":"D4","D4X":"D4X","D4Z":"D4Z","DA":"DC","DC":"WDC","DE":"DEN","DFN":"DFN","DGS":"DGS","DGT":"DGT","DK":"GDK","DRS":"DRS","DRT":"DRT","DVE":"DVE","DVT":"DVT","DX":"DX","DXT":"DXT","DY":"DY","E1S":"E1S","E3G":"E3G","E3T":"E3T","EAD":"EAD","EAT":"EAT","EBN":"EBN","EBR":"EBR","ECD":"ECD","ECW":"ECW","ECZ":"ECK","EDW":"EDW","EGT":"EGT","EHU":"EHF","EIC":"EIC","EID":"EID","EI":"EI","EIT":"EIT","EIY":"EIY","EMB":"EMB","EMD":"EMD","EMT":"EMT","ENB":"ENB","ENK":"ENK","ENY":"ENY","EPL":"EPZ","ESG":"ESG","ES":"ES","ESK":"ESK","ESQ":"ESQ","ESR":"ESR","EST":"EST","ESX":"ESX","ETB":"ETB","ETC":"ETC","ETE":"ETE","ETH":"ETH","ETR":"ETR","ETW":"ETW","EU9":"EU9","EU":"6E","EUS":"EUS","EWV":"EWV","EYB":"EYB","EYT":"EYT","EZ":"EZ","F1S":"F1S","FF":"ZQ","FIT":"FIT","FIX":"FIX","FNG":"FNG","FOB":"FOB","FOL":"FOL","FR":"SFR","FT1":"FT1","FT5":"FT5","FTB":"FTB","FTC":"FTC","FTT":"FTT","FTU":"FTU","FTW":"FTW","FV":"ZF","FYN":"FYN","FYT":"FYT","FYW":"FYW","FZE":"FZE","G0":"G0","G1":"G1","G1K":"G1K","G1N":"G1N","G2":"G2","G4":"G4","G4K":"G4K","G4N":"G4N","G6":"G6","G6K":"G6K","G6N":"G6N","G6X":"G6X","G6Z":"G6Z","GDT":"GDT","GF":"GF","GFT":"GFT","GIE":"GIE","GI":"GD","GIT":"GIT","GN":"GN","H2O":"H2O","H6":"H6","H6X":"H6X","H6Z":"H6Z","H7":"H7","H7X":"H7X","H7Z":"H7Z","HE":"HE","HET":"HET","HFO":"HUF","HQ":"HQ","HRE":"HR","HR":"HR","HRX":"HRX","HRZ":"HRZ","HW":"HW","HWX":"HWX","HWZ":"HWZ","HY":"HY","IBH":"IBH","IBI":"IBI","IBT":"IBT","IBV":"IBV","IC":"IC","IH":"IH","IL":"ILS","ILS":"ILS","IP":"IP","IPO":"IPO","IPT":"IPT","IS":"IS","IST":"IST","IW":"IW","JE":"J7","JPP":"JPP","JY":"6J","K0":"K0","K1":"K1","K2":"K2","K3":"K3","K4":"K4","K5":"K5","K6":"K6","K6K":"K6K","K6N":"K6N","K6S":"K6S","K7":"K7","K7K":"K7K","K7N":"K7N","KAT":"KAT","KAU":"KAU","KC7":"KC7","KC":"KC","KCT":"KCT","KEJ":"KEJ","KEO":"KEO","KEP":"KEP","KGB":"KGB","KGT":"KGT","KJ":"KJ","KJT":"KJT","KKT":"KKT","KMF":"KMF","KMP":"KMP","KNS":"KNS","KOL":"KOL","KOT":"KOT","KP":"KP","KPK":"KPK","KPN":"KPN","KQ":"KQ","KRA":"KRA","KRH":"KRK","KR":"KR","KRK":"KRK","KRN":"KRN","KRW":"KRW","KRZ":"KRZ","KS":"KS","KSN":"KSN","KSV":"KSV","KTT":"KTT","KWB":"KW","KW":"KE","KWK":"KWK","KWN":"KWN","KWT":"KET","KXD":"KX","KY":"KY","KZS":"KZS","KZT":"KZT","KZX":"KZX","KZY":"KZY","LA":"LAX","LBR":"LBR","LE":"LE","LET":"LET","LP":"LP","LPX":"LPX","LPZ":"LPZ","LV":"LAV","M2K":"M2K","M6A":"M6A","M6B":"M6B","M6C":"M6C","M6E":"M6E","M6J":"M6J","M6S":"M6S","MB1":"MB1","MBT":"MBT","MCD":"MCD","MCE":"MCE","MCL":"MCL","MCV":"MCU","MCX":"MCX","ME":"E7","MES":"MES","MET":"MET","MEU":"MEU","MFC":"MFC","MFS":"MFS","MFU":"MFU","MGE":"MGE","MIB":"MIB","MI":"MIA","MIN":"MIN","MIR":"MIR","MKC":"MKC","MLE":"MLE","MMC":"MMC","MME":"MME","MML":"MML","MMM":"MMM","MMN":"MMN","MMR":"MMR","MMW":"MMW","MNH":"MNH","MNQ":"MNQ","MPA":"MPA","MPC":"MPC","MP":"MP","MPP":"MPP","MPT":"MPT","MPU":"MPU","MRG":"MRG","MS1":"MS1","MS4":"MS4","MS5":"MS5","MS6":"MS6","MS7":"MS7","MS8":"MS8","MS9":"MS9","MSC":"MSC","MSF":"MSF","MT1":"MT1","MT3":"MT3","MUC":"MUC","MUN":"MUN","MUS":"MUS","MW":"MWE","MWL":"MWL","MWS":"MWS","MY1":"MY1","MY2":"MY2","MY3":"MY3","MY4":"MY4","MY5":"MY5","MY7":"MY7","MYB":"MYB","MYM":"MYM","N1S":"N1S","NAA":"NAA","NBY":"NBY","NCB":"NCB","NCF":"NCF","NDA":"NDA","NE":"6N","NF":"GNF","NIB":"NIB","NIT":"NIT","NIY":"NIY","NJ":"NJ","NKD":"NKD","NKT":"NKT","NOB":"NOB","NOK":"NOK","NOL":"NOL","NON":"NON","NQB":"NQT","NQ":"NQ","NQQ":"NQQ","NQX":"NQX","NT":"NT","NTW":"NTW","NUB":"NUB","NY":"NYM","NYW":"NYW","O":"ZO","OJ":"OJ","OJT":"OJT","OPF":"OPF","PAC":"PAC","PC":"PC","PJY":"PJY","PK":"PK","PLE":"PLE","PLN":"PLN","PLZ":"PLN","POG":"POG","PRK":"PRK","PSF":"PSF","PS":"PS","PX":"6M","PY":"SY","PZ":"PZ","QA":"QA","QC2":"QC2","QC3":"QC3","QC4":"QC4","QC6":"QC6","QC7":"QC7","QC8":"QC8","QCC":"QCC","QCN":"QCN","QCW":"QCW","QKC":"QKC","QM2":"QM2","QM3":"QM3","QM4":"QM4","QM5":"QM5","QM6":"QM6","QO2":"QO2","QO3":"QO3","QO4":"QO4","QO5":"QO5","QO6":"QO6","QS0":"QS0","QS1":"QS1","QS2":"QS2","QS3":"QS3","QS5":"QS5","QS8":"QS8","QS9":"QS9","QW2":"QW2","QW3":"QW3","QW6":"QW6","QX5":"QX5","R1":"RS1","R1T":"R1T","R2G":"R2G","R2V":"R2V","RA":"6Z","RB":"RMB","RCT":"RST","RDA":"RDA","RE":"RME","RET":"RET","REX":"REX","RFD":"RFD","RF":"RF","RFI":"RFI","RKT":"RKT","RLT":"RLT","RP":"RP","RR":"ZR","RSD":"RSD","RS":"RS","RSG":"RSG","RSI":"RSI","RST":"RGT","RSV":"RSV","RTQ":"RTQ","RTX":"RTX","RTY":"RTY","RU":"6R","RUT":"RVT","RX":"RX","RY":"RY","S1S":"S1S","S7C":"S7C","SAS":"SAS","SB":"SB","SBT":"SBT","SDA":"SDA","SD":"SDG","SDI":"SDI","SEK":"SEK","S":"ZS","SF":"6S","SFI":"SF","SG":"SG","SGT":"SGT","SIR":"SIR","SJY":"SJY","SMC":"SMC","SM":"ZM","SMT":"SMT","SON":"SON","SOT":"SBT","SOX":"SOX","SQ2":"SQ2","SQ5":"SQ5","SR1":"SR1","SR3":"SR3","SU":"SU","SUT":"SUT","SWT":"SWT","SXB":"SXB","SXI":"SXI","SXO":"SXO","SXR":"SXR","SXT":"SXT","T1S":"T1S","TAF":"TAF","TBF":"TBF","TBM":"TBM","TBT":"TBT","TEX":"TEX","TFY":"TFY","THW":"THW","TIE":"TIE","TN":"TN","TNT":"TNT","TOB":"TOB","TOF":"TOF","TOS":"SOT","TOU":"TOU","TOW":"TOW","TOX":"TOX","TPB":"TPB","TPD":"TPD","TPT":"TPT","TPY":"TPY","TRB":"TRB","TRI":"TRI","TRL":"TRL","TRM":"TRM","TRW":"TRW","TTW":"TTW","TUB":"TUB","TUF":"TUF","TU":"ZT","TUL":"TUL","TUT":"TUT","TUX":"TUX","TUY":"TUN","TWB":"TWB","TWE":"TWE","TWU":"TWU","TWW":"TWW","TY":"ZN","TYT":"TYT","TYW":"TYW","TYX":"TYX","UB":"UB","UBT":"UBT","UFB":"UFB","UFE":"UFE","UFV":"UFV","UNO":"UNO","US":"ZB","USS":"USS","VC":"VC","VMT":"VMT","VU":"VU","VX1":"VX1","VX2":"VX2","VX4":"VX4","VX5":"VX5","VX":"VX","VXM":"VXM","W":"ZW","WMK":"WMK","WQ6":"WQ6","XAB":"XAB","XAE":"XAE","XAF":"XAF","XAI":"XAI","XAK":"XAK","XAP":"XAP","XAR":"XAR","XAU":"XAU","XAV":"XAV","XAY":"XAY","XAZ":"XAZ","XCT":"XVT","XET":"XET","XFT":"XFT","XIT":"XIT","XKT":"XKT","XMT":"XBT","XPT":"XPT","XTT":"XRT","XUT":"XUT","XVT":"VXT","XYT":"XYT","XZT":"XZT","YA":"YA","YC":"XC","YIA":"YIA","YIB":"YIB","YIC":"YIC","YID":"YID","YIE":"YIE","YII":"YII","YIL":"YIL","YIO":"YIO","YIT":"YIT","YIW":"YIW","YIY":"YIY","YK":"XK","YM":"YM","YMT":"YMT","YMX":"YMX","YW":"XW","YZ":"YZ","Z3W":"Z3W","ZAR":"ZAR","ZBT":"ZBT","ZBW":"ZBW","ZCT":"ZCT","ZFT":"ZFT","ZJ":"ZJ","ZLT":"ZLT","ZMT":"ZMT","ZNS":"ZNS","ZR":"ZR","ZTT":"ZTT","ZTW":"ZTW","ZWC":"ZWC","ZWT":"ZWT","AEX":"FTI","AFB":"UB","ALJ":"ALS","ALM":"ALM","AM":"AM","ARB":"ARB","ARS":"ARS","ATX":"ATX","AUD":"AUD","AXA":"AXA","AXF":"AXF","AXJ":"AXJ","AXL":"AXL","AXM":"AXM","AXQ":"AXQ","AXS":"AXS","AXV":"AXV","BB":"JB","BBN":"BB","BD":"GBL","BFX":"BXF","BGI":"BGI","BL":"GBM","BN":"BN","BON":"BON","BQ":"BQ","BRI":"BRI","BS":"BS","BTM":"BTM","BTP":"BTP","BTS":"BTS","BTU":"BTU","BV":"BV","BX":"GBX","C1L":"C1L","C3F":"C3F","C5F":"C5F","C7F":"C7F","CAG":"CAG","CA":"EMA","CCM":"CCM","CCO":"CCO","CEE":"CEE","CEN":"CEN","CEX":"CEX","CGR":"CGR","CHF":"CHF","CIN":"CIN","CLI":"CLI","CLP":"CLP","CN":"CN","CNI":"CNY","CO":"CON","COP":"COP","COR":"COR","CPE":"CPE","CPR":"CPR","CRD":"WBS","CSO":"CSO","CWF":"CWF","CXA":"CXA","CXB":"CXB","CXE":"CXE","CXI":"CXI","CXL":"CXL","CXP":"CXP","CXR":"CXR","CXS":"CXS","CXT":"CXT","D3D":"D3D","DDI":"DDI","DIJ":"DI1","DIV":"DIV","DOL":"DOL","DVD":"DVD","DXD":"DXD","DXM":"DXM","DXS":"DXS","EA":"EA","EAL":"MEA","EAN":"BEA","EBD":"EBD","EBE":"EU3","EB":"BRN","EBF":"EBF","EBI":"EBD","EBT":"BRT","ECX":"ECX","EDV":"EDV","EED":"EED","EE":"EE","EFD":"EFD","EF":"AFR","EFQ":"AFQ","EFS":"AFS","EFY":"AFY","EMA":"EMA","EPE":"EPE","EPO":"EPO","EPR":"EPR","ESA":"ESG","ESC":"ESC","ESE":"ESE","ESF":"ESF","ESJ":"ESJ","ESL":"ESL","ESM":"ESM","ESN":"ESN","ESO":"ESO","ESP":"ESP","ESS":"ESS","ESU":"ESU","ESV":"ESV","ESW":"ESW","ESZ":"ESZ","ETD":"ETD","ET":"ATW","ETQ":"ATQ","ETS":"ATS","ETY":"ATY","EU3":"EU3","EUD":"EUD","EUR":"EUR","EXD":"EXD","EX":"ESX","EXZ":"XZ","FCG":"FCG","FCT":"FCT","FID":"FID","FIN":"FIN","FMX":"FMX","FMY":"FMY","FNA":"FNA","FND":"FND","FN":"GWM","FNQ":"NGQ","FNS":"NGS","FNY":"FNY","FOX":"FOX","FPD":"FPD","FPH":"FPH","FRC":"FRC","FTH":"FTH","FVN":"FVN","FVS":"FVS","G":"G02","GAS":"G","GB3":"GB3","GBH":"GBH","GBO":"GBO","GBP":"GBP","GBQ":"GBP","GBW":"GBW","GDI":"GDI","GDS":"GDS","GDV":"GDV","GGI":"GGI","GLD":"GLD","GOL":"GOL","GT":"GT","GWT":"GWT","GWY":"GWY","GX":"GX","H":"H","HOS":"HOS","HOU":"HOU","HSI":"HSI","HT":"HT","HWF":"HWF","I":"I02","IB":"IB","ICF":"ICF","IE":"I","IET":"IT","IHO":"UHO","IND":"IND","IRB":"UHU","IRBT":"LRT","IR":"IR","ISE":"ISE","ISP":"ISP","JG":"JG","JPI":"JPY","JRT":"JRT","KAN":"KAN","KLI":"KLI","KPO":"CPO","LCE":"LCE","LCP":"LCP","LF":"Z","LG":"R","LGT":"RT","LHT":"LHT","LHW":"USW","LIV":"USO","LJN":"USP","LLR":"FEF","LLS":"FEO","LRC":"RC","LRCT":"RCT","LU":"TWS","LY":"Y2","M":"M70","MAE":"MAE","MAS":"MAS","MAW":"MAW","MAX":"MFA","MBE":"MBE","MBZ":"MBZ","MC2":"MCP","MCH":"MCH","MDK":"MDK","MDM":"MDM","MEA":"MEA","MEE":"MEE","MEF":"MEF","MEL":"MEL","MEM":"MEM","MEN":"MEN","MEP":"MEP","MFI":"MFI","MFR":"MFR","MGA":"MGA","MGC":"MGC","MGS":"MGS","MIX":"MIX","MJP":"MJP","MKU":"MKU","MMY":"MMY","MNL":"MNL","MNO":"MNA","MNS":"MNS","MNW":"MNW","MSD":"MSD","MSG":"MSG","MSI":"MSI","MSJ":"MSJ","MSP":"MSP","MSR":"MSM","MSS":"MSS","MST":"MST","MSU":"MSU","MSW":"MSW","MSZ":"MSZ","MT":"FCE","MTH":"MTH","MTW":"MTW","MWA":"WM","MWB":"MWB","MWC":"MWC","MWD":"MWD","MWF":"MWF","MWH":"MWH","MWI":"MWI","MWN":"MWN","MWO":"MWO","MWP":"MWP","MWQ":"MWQ","MWR":"MWR","MWT":"MWT","MXB":"MXB","MXG":"MXG","MXN":"MXN","MZA":"MZA","NAU":"NAU","NCQ":"NCQ","NCY":"NCY","NDI":"NDI","NEA":"NEA","NID":"NID","NN":"NK","NPH":"NPH","NTH":"NTH","NU":"NU","NXJ":"NXJ","NZD":"NZD","OAM":"OAM","OAT":"OAT","OE":"O","OFR1":"SF1","OFR3":"SF3","OIL":"OIL","OP":"P","PG":"ECO","PLA":"PLA","PM":"EBM","PNS":"PN","POL":"POL","PQ":"PQ","PSI":"PSI","PVF":"PVF","PV":"PV","PWF":"PWF","QC":"C","QCT":"LCT","QK":"T","QUME":"UME","QW":"W","QWT":"WT","RED":"RED","RES":"RES","SA":"ESA","SBD":"SBD","SCE":"SCE","SCI":"SCI","SCP":"SCP","SDX":"SDX","SED":"SED","SEE":"STE","SEG":"SEG","SID":"SID","SIN":"ESI","SJC":"SJC","SJ":"EST","SK":"ESH","SLC":"SLC","SLI":"SLI","SLS":"SLS","SLV":"SLV","SMD":"SMD","SMM":"SMM","SMS":"SMS","SMX":"SMX","SND":"ND","SNS":"NS","SO3":"SO3","SOA":"SOA","SOY":"SOY","SPI":"AP","SQ":"STX","SS":"SGP","SSX":"SSX","ST3":"ST3","STD":"STD","STF":"STF","STG":"STG","STI":"ST","STJ":"STJ","STL":"STL","STM":"STM","STN":"STI","STO":"STO","STP":"STP","STQ":"STQ","STS":"STS","STU":"STU","STW":"STW","SUD":"SUD","SUN":"SUN","SUS":"SUS","SVF":"SVF","SWF":"SWF","SW":"SMI","SXE":"SXE","SX":"ESB","SY":"ESY","TA":"STB","TC":"STC","TDD":"TDD","TD":"TDX","TF":"STA","TIN":"TIN","TJ":"STH","TL":"STV","TM":"STN","TP":"STZ","TR":"STR","TT":"STT","TUK":"TUK","TWN":"TWN","TX":"STY","UAA":"UAA","UAL":"UAL","UAM":"UAM","UAQ":"UAQ","UAS":"UAS","UAV":"UAV","UBL":"UBL","UBQ":"UBQ","U":"U","UKA":"UKA","UKD":"UKD","UNF":"UNF","UPL":"UPL","UPO":"UPO","US3":"US3","UT":"UT","UZQ":"UZQ","WAW":"WK","WBT":"WBT","WDO":"WDO","WEA":"WEA","WEU":"WEU","WIN":"WIN","WMA":"WMA","WPD":"WPD","WTO":"WTO","XFC":"XFC","XG":"DAX","XGL":"XGL","XSF":"XSF","XT":"XT","XXE":"XXE","XXP":"XXP","XXS":"XXS","YMA":"YMA","YT":"YT"} \ No newline at end of file +{"1AEX":"1FT","1MT":"1FC","2AEX":"2FT","2EW":"2EW","2MT":"2FC","4AEX":"4FT","4MT":"4FC","67T":"67T","10Y":"10Y","1D0":"1D0","1D1":"1D1","1D2":"1D2","1D4":"1D4","1G6":"1G6","1H0":"1H0","1H1":"1H1","1H2":"1H2","1H3":"1H3","1H4":"1H4","1H5":"1H5","1H6":"1H6","1H7":"1H7","1HQ":"1HQ","1HR":"1HR","1HS":"1HS","1HW":"1HW","1LP":"1LP","20U":"20U","25U":"25U","2GT":"2GT","2K0":"2K0","2K1":"2K1","2K2":"2K2","2K3":"2K3","2K4":"2K4","2K5":"2K5","2K6":"2K6","2K7":"2K7","2KP":"2KP","2KQ":"2KQ","2KR":"2KR","2KS":"2KS","2KW":"2KW","2VT":"2VT","2YY":"2YY","30U":"30U","30Y":"30Y","33K":"33K","35U":"35U","3G0":"3G0","3G1":"3G1","3G2":"3G2","3G4":"3G4","3G6":"3G6","3K0":"3K0","3K1":"3K1","3K2":"3K2","3K3":"3K3","3K4":"3K4","3K5":"3K5","3K6":"3K6","3K7":"3K7","3KP":"3KP","3KQ":"3KQ","3KS":"3KS","3KW":"3KW","3N":"Z3N","40U":"40U","45U":"45U","4D0":"4D0","4D1":"4D1","4D2":"4D2","4D4":"4D4","4G6":"4G6","4H0":"4H0","4H1":"4H1","4H2":"4H2","4H3":"4H3","4H4":"4H4","4H5":"4H5","4H6":"4H6","4H7":"4H7","4HQ":"4HQ","4HR":"4HR","4HS":"4HS","4HW":"4HW","4LP":"4LP","50U":"50U","55U":"55U","5YY":"5YY","60U":"60U","65U":"65U","6EB":"6EB","70U":"70U","A2R":"A2R","A2T":"A2T","ABB":"ABB","ACD":"ACD","AD":"6A","ADR":"ADR","ADT":"ADT","AE":"AW","AFR":"AFR","AFT":"AFT","AHB":"AHB","AJY":"AJY","AMB":"AMB","ANE":"ANE","AQR":"AQR","AQT":"AQT","AR":"AR","ARR":"ARR","ART":"ART","AS":"AS","ASI":"ASI","ASN":"ASN","ASR":"ASR","AST":"AST","ATB":"ATB","AUW":"AUW","AWN":"AWN","AWT":"AWT","B1S":"B1S","BAG":"BAG","BAT":"BAT","BCX":"BCX","BEN":"BEN","BET":"BET","BF":"SS","BGR":"BGR","BIO":"BIO","BIT":"BIT","BLI":"BLI","BLK":"BLK","BLT":"BLT","BME":"BME","BMT":"BMT","BNB":"BNB","BO":"ZL","BOT":"BOT","BPE":"BPE","BP":"6B","BPR":"BPR","BPT":"BPT","BR":"6L","BST":"BST","BTB":"BTB","BTC":"BTC","BTE":"BTE","BT":"BOS","BTG":"BGT","C3S":"C3S","CAD":"CAD","CB":"CB","CCB":"CCT","CC":"CC","CCI":"CCI","CCT":"CCT","CD":"6C","C":"ZC","CH":"CHI","CHP":"CHP","CJY":"CJY","CKO":"CZK","CKS":"CKS","CNH":"CNH","CPO":"CPO","CPV":"CPV","CSC":"CSC","CT":"CT","CTT":"CTT","CU":"CUS","CVB":"CVB","CW1":"CW1","CWD":"CWD","CWR":"CWR","D0":"D0","D1":"D1","D1X":"D1X","D1Z":"D1Z","D2":"D2","D4":"D4","D4X":"D4X","D4Z":"D4Z","DA":"DC","DC":"WDC","DE":"DEN","DFN":"DFN","DGS":"DGS","DGT":"DGT","DHB":"DHB","DHY":"DHY","DK":"GDK","DRS":"DRS","DRT":"DRT","DVE":"DVE","DVT":"DVT","DX":"DX","DXT":"DXT","DY":"DY","DYT":"DYT","E1S":"E1S","E3G":"E3G","E3T":"E3T","EAD":"EAD","EBM":"EBM","EBR":"EBR","ECD":"ECD","ECZ":"ECK","EEM":"EEM","EGT":"EGT","EHU":"EHF","EI":"EI","EIT":"EIT","EMB":"EMB","EMD":"EMD","EMT":"EMT","EMV":"EMV","ENB":"ENB","ENK":"ENK","ENY":"ENY","ENZ":"ENZ","EPL":"EPZ","ESG":"ESG","ES":"ES","ESK":"ESK","ESQ":"ESQ","ESR":"ESR","EST":"EST","ESX":"ESX","ETB":"ETB","ETE":"ETE","ETH":"ETH","ETR":"ETR","EU9":"EU9","EU":"6E","EUS":"EUS","EWE":"EWE","EWF":"EWF","EWM":"EWM","EWT":"EWT","EYB":"EYB","EZ":"EZ","F1S":"F1S","FF":"ZQ","FFV":"FFV","FNG":"FNG","FR":"SFR","FT1":"FT1","FT5":"FT5","FTB":"FTB","FTC":"FTC","FTT":"FTT","FTU":"FTU","FV":"ZF","G":"G02","G0":"G0","G1":"G1","G1K":"G1K","G1N":"G1N","G2":"G2","G4":"G4","G4K":"G4K","G4N":"G4N","G6":"G6","G6K":"G6K","G6N":"G6N","G6X":"G6X","G6Z":"G6Z","GDT":"GDT","GF":"GF","GFT":"GFT","GIE":"GIE","GI":"GD","GIT":"GIT","GN":"GN","H2O":"H2O","H6":"H6","H6X":"H6X","H6Z":"H6Z","H7":"H7","H7X":"H7X","H7Z":"H7Z","HE":"HE","HET":"HET","HFO":"HUF","HQ":"HQ","HRE":"HR","HR":"HR","HRS":"HRS","HRX":"HRX","HRZ":"HRZ","HW":"HW","HWX":"HWX","HWZ":"HWZ","HYB":"HYB","HY":"HY","HYT":"HYT","IBH":"IBH","IBI":"IBI","IBV":"IBV","IDR":"IDR","IEM":"IEM","IL":"ILS","ILS":"ILS","IPO":"IPO","IPT":"IPT","IQB":"IQB","IQT":"IQT","IST":"IST","JE":"J7","JPP":"JPP","JY":"6J","K0":"K0","K1":"K1","K2":"K2","K3":"K3","K4":"K4","K5":"K5","K6":"K6","K6K":"K6K","K6N":"K6N","K6S":"K6S","K7":"K7","K7K":"K7K","K7N":"K7N","KAT":"KAT","KAU":"KAU","KC7":"KC7","KC":"KC","KCT":"KCT","KEJ":"KEJ","KEO":"KEO","KEP":"KEP","KGB":"KGB","KGT":"KGT","KJ":"KJ","KJT":"KJT","KKT":"KKT","KMF":"KMF","KMP":"KMP","KNS":"KNS","KOL":"KOL","KOT":"KOT","KP":"KP","KPK":"KPK","KPN":"KPN","KQ":"KQ","KRA":"KRA","KRH":"KRK","KR":"KR","KRK":"KRK","KRN":"KRN","KRW":"KRW","KRZ":"KRZ","KS":"KS","KSN":"KSN","KSV":"KSV","KTT":"KTT","KWB":"KW","KWD":"KWD","KW":"KE","KWK":"KWK","KWN":"KWN","KWT":"KET","KXD":"KX","KY":"KY","KZS":"KZS","KZT":"KZT","KZX":"KZX","KZY":"KZY","LA":"LAX","LBR":"LBR","LE":"LE","LET":"LET","LP":"LP","LPX":"LPX","LPZ":"LPZ","LV":"LAV","M2K":"M2K","M6A":"M6A","M6B":"M6B","M6E":"M6E","MBT":"MBT","MCD":"MCD","MCE":"MCE","MCL":"MCL","MCV":"MCU","MCX":"MCX","ME":"E7","MES":"MES","MET":"MET","MEU":"MEU","MFC":"MFC","MFS":"MFS","MFU":"MFU","MFV":"MFV","MGE":"MGE","MIB":"MIB","MIE":"MIE","MI":"MIA","MIN":"MIN","MIR":"MIR","MIU":"MIU","MJY":"MJY","MKC":"MKC","MLE":"MLE","MMC":"MMC","MME":"MME","MML":"MML","MMM":"MMM","MMN":"MMN","MMR":"MMR","MMW":"MMW","MNH":"MNH","MNI":"MNI","MNK":"MNK","MNQ":"MNQ","MPA":"MPA","MP":"MP","MPP":"MPP","MPT":"MPT","MPU":"MPU","MRE":"MGE","MRG":"MRG","MSC":"MSC","MSF":"MSF","MSL":"MSL","MSU":"MSU","MTN":"MTN","MUC":"MUC","MUJ":"MUJ","MUK":"MUK","MUL":"MUL","MUN":"MUN","MUO":"MUO","MUP":"MUP","MUS":"MUS","MVM":"MVM","MVW":"MVW","MW":"MWE","MWL":"MWL","MWN":"MWN","MWS":"MWS","MXA":"MXA","MXE":"MXE","MXJ":"MXJ","MXP":"MXP","MYB":"MYB","MYM":"MYM","MZC":"MZC","MZL":"MZL","MZM":"MZM","MZS":"MZS","MZW":"MZW","N1S":"N1S","NAA":"NAA","NCF":"NCF","NDA":"NDA","NE":"6N","NF":"GNF","NIT":"NIT","NIY":"NIY","NJ":"NJ","NJY":"NJY","NKD":"NKD","NKT":"NKT","NOK":"NOK","NQB":"NQT","NQ":"NQ","NQQ":"NQQ","NQX":"NQX","NSK":"NSK","NT":"NT","NY":"NYM","NZC":"NZC","O":"ZO","OJ":"OJ","OJT":"OJT","OPF":"OPF","OSF":"OSF","PAC":"PAC","PAD":"PAD","PCD":"PCD","PC":"PC","PJY":"PJY","PK":"PK","PLE":"PLE","PLN":"PLN","PLZ":"PLN","PNK":"PNK","POG":"POG","PRK":"PRK","PSF":"PSF","PS":"PS","PSK":"PSK","PX":"6M","PY":"SY","PZ":"PZ","QA":"QA","QBT":"QBT","QC2":"QC2","QC3":"QC3","QC4":"QC4","QC6":"QC6","QC7":"QC7","QC8":"QC8","QCC":"QCC","QCN":"QCN","QCW":"QCW","QDF":"QDF","QDM":"QDM","QDO":"QDO","QEF":"QEF","QEM":"QEM","QET":"QET","QKC":"QKC","QM2":"QM2","QM3":"QM3","QM4":"QM4","QM5":"QM5","QM6":"QM6","QND":"QND","QNF":"QNF","QNM":"QNM","QO2":"QO2","QO3":"QO3","QO4":"QO4","QO5":"QO5","QO6":"QO6","QRF":"QRF","QRM":"QRM","QRT":"QRT","QS0":"QS0","QS1":"QS1","QS2":"QS2","QS3":"QS3","QS5":"QS5","QS9":"QS9","QSF":"QSF","QSM":"QSM","QSP":"QSP","QW2":"QW2","QW3":"QW3","QW6":"QW6","QX5":"QX5","R1":"RS1","R1T":"R1T","R2G":"R2G","R2V":"R2V","RA":"6Z","RB":"RMB","RCT":"RST","RDA":"RDA","RE":"RME","RET":"RET","REX":"REX","RFD":"RFD","RF":"RF","RFI":"RFI","RKT":"RKT","RLB":"RLB","RLT":"RLT","RNB":"RNB","RP":"RP","RR":"ZR","RSD":"RSD","RS":"RS","RSG":"RSG","RSI":"RSI","RSO":"RSO","RST":"RGT","RSV":"RSV","RTQ":"RTQ","RTX":"RTX","RTY":"RTY","RUT":"RVT","RX":"RX","RY":"RY","S1S":"S1S","S7C":"S7C","SAS":"SAS","SB":"SB","SBT":"SBT","SDA":"SDA","SD":"SDG","SDI":"SDI","SEK":"SEK","S":"ZS","SF":"6S","SFI":"SF","SGD":"SGD","SGT":"SGT","SG":"SG","SIR":"SIR","SJY":"SJY","SMC":"SMC","SM":"ZM","SMT":"SMT","SOL":"SOL","SON":"SON","SOT":"SBT","SOX":"SOX","SPR":"SPR","SPT":"SPT","SQ2":"SQ2","SQ5":"SQ5","SR1":"SR1","SR3":"SR3","SUT":"SUT","SU":"SU","SWT":"SWT","SXB":"SXB","SXI":"SXI","SXO":"SXO","SXR":"SXR","SXT":"SXT","SYP":"SYP","T1S":"T1S","TAF":"TAF","TBF":"TBF","TBM":"TBM","TBT":"TBT","TEM":"TEM","TET":"TET","TEX":"TEX","TFY":"TFY","THA":"THA","THW":"THW","TI3":"TI3","TIE":"TIE","TN":"TN","TNT":"TNT","TOB":"TOB","TOF":"TOF","TOS":"SOT","TOU":"TOU","TOW":"TOW","TOX":"TOX","TPB":"TPB","TPD":"TPD","TPT":"TPT","TPY":"TPY","TRB":"TRB","TRI":"TRI","TRL":"TRL","TRM":"TRM","TRW":"TRW","TTW":"TTW","TUB":"TUB","TUF":"TUF","TU":"ZT","TUL":"TUL","TUT":"TUT","TUX":"TUX","TUY":"TUN","TWB":"TWB","TWE":"TWE","TWU":"TWU","TWW":"TWW","TY":"ZN","TYT":"TYT","TYW":"TYW","TYX":"TYX","UB":"UB","UBT":"UBT","UFB":"UFB","UFE":"UFE","UFV":"UFV","US":"ZB","USS":"USS","VA":"VA","VC":"VC","VMT":"VMT","VU":"VU","VX1":"VX1","VX2":"VX2","VX4":"VX4","VX5":"VX5","VX":"VX","VXM":"VXM","W":"ZW","WK1":"KW1","WMK":"WMK","WQ6":"WQ6","XAB":"XAB","XAE":"XAE","XAF":"XAF","XAI":"XAI","XAK":"XAK","XAP":"XAP","XAR":"XAR","XAU":"XAU","XAV":"XAV","XAY":"XAY","XAZ":"XAZ","XBT":"XBT","XCT":"XVT","XET":"XET","XEU":"XEU","XFT":"XFT","XIT":"XIT","XKT":"XKT","XLB":"XLB","XMT":"XBT","XNB":"XNB","XPT":"XPT","XRP":"XRP","XTT":"XRT","XUT":"XUT","XVT":"VXT","XY3":"XY3","XYR":"XYR","XYT":"XYT","XZT":"XZT","YA":"YA","YC":"XC","YIA":"YIA","YIB":"YIB","YIC":"YIC","YID":"YID","YIE":"YIE","YII":"YII","YIL":"YIL","YIO":"YIO","YIT":"YIT","YIW":"YIW","YIY":"YIY","YK":"XK","YM":"YM","YMT":"YMT","YMX":"YMX","YW":"XW","YZ":"YZ","Z3W":"Z3W","ZBT":"ZBT","ZBW":"ZBW","ZCT":"ZCT","ZFT":"ZFT","ZJ":"ZJ","ZLT":"ZLT","ZMT":"ZMT","ZNS":"ZNS","ZR":"ZR","ZTT":"ZTT","ZTW":"ZTW","ZWC":"ZWC","ZWT":"ZWT","AEX":"FTI","AFB":"UB","AFS":"AFS","ALJ":"ALS","ALM":"ALM","AM":"AM","ARB":"ARB","ARS":"ARS","ASF":"ESF","ATX":"ATX","AUD":"AUD","AUT":"AUS","AXF":"AXF","AXJ":"AXJ","BB":"JB","BBN":"BB","BD":"GBL","BFX":"BXF","BGI":"BGI","BKS":"BKS","BL":"GBM","BMS":"BMS","BN":"BN","BON":"BON","BQ":"BQ","BRI":"BRI","BS":"BS","BTM":"BTM","BTP":"BTP","BTS":"BTS","BTU":"BTU","BV":"BV","BWS":"BCS","BX":"GBX","C3F":"C3F","C5F":"C5F","C7F":"C7F","CAG":"CAG","CA":"EMA","CAN":"CAN","CCM":"CCM","CCO":"CCO","CEE":"CEE","CEN":"CEN","CGR":"CGR","CHF":"CHF","CHL":"CHL","CIN":"CIN","CLI":"CLI","CLP":"CLP","CN":"CN","CO":"CON","COP":"COP","COR":"COR","CPE":"CPE","CPR":"CPR","CRD":"WBS","CSO":"CSO","CTO":"CTO","CWF":"CWF","CXA":"CXA","CXB":"CXB","CXE":"CXE","CXI":"CXI","CXL":"CXL","CXP":"CXP","CXR":"CXR","CXS":"CXS","CXT":"CXT","D3D":"D3D","DBI":"DBI","DDI":"DDI","DIJ":"DI1","DIV":"DIV","DOL":"DOL","DXM":"DXM","DXS":"DXS","EA":"EA","EAL":"MEA","EAN":"BEA","EBD":"EBD","EBE":"EU3","EB":"BRN","EBF":"EBF","EBI":"EBD","EBT":"BRT","ECX":"ECX","EDE":"EDE","EDV":"EDV","EDW":"EDW","EED":"EED","EE":"EE","EEU":"EEU","EFD":"EFD","EF":"AFR","EFQ":"AFQ","EFS":"AFS","EFY":"AFY","EID":"EID","EJP":"EJP","EMA":"EMA","EPE":"EPE","EPO":"EPO","EPR":"EPR","ER3":"ER3","ESA":"ESG","ESC":"ESC","ESE":"ESE","ESF":"ESF","ESJ":"ESJ","ESL":"ESL","ESM":"ESM","ESN":"ESN","ESO":"ESO","ESP":"ESP","ESS":"ESS","ESU":"ESU","ESV":"ESV","ESW":"ESW","ESZ":"ESZ","ETD":"ETD","ET":"ATW","ETQ":"ATQ","ETS":"ATS","ETY":"ATY","EU3":"EU3","EUD":"EUD","EUK":"EUK","EUP":"EUP","EUR":"EUR","EXD":"EXD","EX":"ESX","EXZ":"XZ","FCG":"FCG","FCT":"FCT","FID":"FID","FIN":"FIN","FIT":"FIT","FMX":"FMX","FMY":"FMY","FNA":"FNA","FND":"FND","FN":"GWM","FNQ":"NGQ","FNS":"NGS","FNY":"FNY","FOX":"FOX","FPD":"FPD","FPH":"FPH","FRC":"FRC","FSO":"SOY","FTH":"FTH","FVN":"FVN","FVS":"FVS","GAS":"G","GB3":"GB3","GBH":"GBH","GBO":"GBO","GBP":"GBP","GBQ":"GBP","GBR":"GBR","GBW":"GBW","GDI":"GDI","GDS":"GDS","GDV":"GDV","GGI":"GGI","GLD":"GLD","GOL":"GOL","GT":"GT","GWT":"GWT","GWY":"GWY","GX":"GX","HOS":"HOS","HOU":"HOU","HSI":"HSI","HT":"HT","H":"H","HWF":"HWF","I":"I02","IB":"IB","ICF":"ICF","IE":"I","IET":"IT","IHO":"UHO","IND":"IND","IRB":"UHU","IRBT":"LRT","IR":"IR","ISE":"ISE","ISP":"ISP","JAP":"JAP","JG":"JG","JPI":"JPY","JRT":"JRT","KAN":"KAN","KLI":"KLI","KPO":"CPO","LCE":"LCE","LCP":"LCP","LF":"Z","LG":"R","LGT":"RT","LHT":"LHT","LHW":"USW","LIV":"USO","LJN":"USP","LLR":"FEF","LLS":"FEO","LRC":"RC","LRCT":"RCT","LU":"TWS","LY":"Y2","M":"M70","MAE":"MAE","MAS":"MAS","MAW":"MAW","MAX":"MFA","MBE":"MBE","MBZ":"MBZ","MC2":"MCP","MCH":"MCH","MDK":"MDK","MDM":"MDM","MEA":"MEA","MEE":"MEE","MEF":"MEF","MEL":"MEL","MEM":"MEM","MEN":"MEN","MEP":"MEP","MEX":"MEX","MFI":"MFI","MFR":"MFR","MGA":"MGA","MGC":"MGC","MGS":"MGS","MJP":"MJP","MKU":"MKU","MMY":"MMY","MNL":"MNL","MNO":"MNA","MNW":"MNW","MRM":"MRM","MRQ":"MRQ","MRW":"MRW","MSA":"MSA","MSD":"MSD","MSI":"MSI","MSJ":"MSJ","MSP":"MSP","MSR":"MSM","MSS":"MSS","MST":"MST","MSW":"MSW","MSZ":"MSZ","MT":"FCE","MTH":"MTH","MTW":"MTW","MWA":"WM","MWB":"MWB","MWC":"MWC","MWD":"MWD","MWF":"MWF","MWH":"MWH","MWI":"MWI","MWO":"MWO","MWP":"MWP","MWQ":"MWQ","MWR":"MWR","MWT":"MWT","MXG":"MXG","MXN":"MXN","MZA":"MZA","NAU":"NAU","NCQ":"NCQ","NCY":"NCY","NDI":"NDI","NEA":"NEA","NID":"NID","NN":"NK","NPH":"NPH","NTH":"NTH","NTW":"NTW","NU":"NU","NXJ":"NXJ","NZD":"NZD","NZL":"NZL","OAM":"OAM","OAT":"OAT","OE":"O","OFR1":"SF1","OFR3":"SF3","OIL":"OIL","OP":"P","PG":"ECO","PLA":"PLA","PM":"EBM","PNS":"PN","POL":"POL","PQ":"PQ","PSI":"PSI","PVF":"PVF","PV":"PV","PWF":"PWF","QC":"C","QCT":"LCT","QK":"T","QUME":"UME","QW":"W","QWT":"WT","RED":"RED","RES":"RES","SA3":"SA3","SA":"ESA","SBD":"SBD","SCE":"SCE","SCI":"SCI","SCP":"SCP","SDX":"SDX","SED":"SED","SEE":"STE","SEG":"SEG","SID":"SID","SIN":"ESI","SJC":"SJC","SJ":"EST","SK":"ESH","SLI":"SLI","SLS":"SLS","SLV":"SLV","SMD":"SMD","SMM":"SMM","SMS":"SMS","SMX":"SMX","SND":"ND","SNS":"NS","SO3":"SO3","SOA":"SOA","SPI":"AP","SQ":"STX","SRI":"SRI","SSE":"SSE","SS":"SGP","SSX":"SSX","ST3":"ST3","STD":"STD","STF":"STF","STG":"STG","STI":"ST","STJ":"STJ","STL":"STL","STM":"STM","STN":"STI","STO":"STO","STP":"STP","STQ":"STQ","STS":"STS","STU":"STU","STW":"STW","SU1":"SU1","SU2":"SU2","SU3":"SU3","SU5":"SU5","SUD":"SUD","SUN":"SUN","SUS":"SUS","SWF":"SWF","SW":"SMI","SWI":"SWI","SXE":"SXE","SX":"ESB","SY":"ESY","TA":"STB","TC":"STC","TDD":"TDD","TD":"TDX","TF":"STA","TIN":"TIN","TJ":"STH","TL":"STV","TM":"STN","TP":"STZ","TR":"STR","TT":"STT","TUK":"TUK","TWN":"TWN","TX":"STY","UAA":"UAA","UAL":"UAL","UAM":"UAM","UAQ":"UAQ","UAS":"UAS","UAV":"UAV","UBL":"UBL","UBQ":"UBQ","U":"U","UKA":"UKA","UKD":"UKD","UNF":"UNF","UPL":"UPL","UPO":"UPO","US3":"US3","UT":"UT","UZQ":"UZQ","WBT":"WBT","WDO":"WDO","WEA":"WEA","WEU":"WEU","WIN":"WIN","WMA":"WMA","WPD":"WPD","WPT":"WPT","WTO":"WTO","XFC":"XFC","XG":"DAX","XGL":"XGL","XSF":"XSF","XT":"XT","XXE":"XXE","XXP":"XXP","XXS":"XXS","YMA":"YMA","YPT":"YPT","YT":"YT","ZAR":"ZAR"} \ No newline at end of file From c88730815bc0d44fbf2ef0985ecd124794f55cd9 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 28 Jul 2025 23:04:08 +0300 Subject: [PATCH 02/25] refactor: load Future symbol feat: add normalization of Future symbol to LEan format --- .../IQFeedDataQueueUniverseProvider.cs | 83 +++++++++++++++---- 1 file changed, 65 insertions(+), 18 deletions(-) diff --git a/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs b/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs index c8f11b1..a4d93de 100644 --- a/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs +++ b/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs @@ -96,8 +96,7 @@ public IQFeedDataQueueUniverseProvider() /// IQFeed ticker public string GetBrokerageSymbol(Symbol symbol) { - string leanSymbol; - return _symbols.TryGetValue(symbol, out leanSymbol) ? leanSymbol : string.Empty; + return _symbols.TryGetValue(symbol, out var brokerageSymbol) ? brokerageSymbol : string.Empty; } /// @@ -387,13 +386,13 @@ private IEnumerable LoadSymbols() if (columns[columnSymbol].EndsWith("#") || columns[columnSymbol].EndsWith("#C") || columns[columnSymbol].EndsWith("$$")) continue; - var futuresTicker = columns[columnSymbol].TrimStart(new[] { '@' }); + var futuresTicker = NormalizeFuturesTicker(columns[columnExchange], columns[columnSymbol]); var parsed = SymbolRepresentation.ParseFutureTicker(futuresTicker); var underlyingString = parsed.Underlying; - if (_iqFeedNameMap.ContainsKey(underlyingString)) - underlyingString = _iqFeedNameMap[underlyingString]; + if (_iqFeedNameMap.TryGetValue(underlyingString, out var underlying)) + underlyingString = underlying; else { if (!mapExists) @@ -412,24 +411,44 @@ private IEnumerable LoadSymbols() } var market = GetFutureMarket(underlyingString, columns[columnExchange]); - canonicalSymbol = Symbol.Create(underlyingString, SecurityType.Future, market); - if (!symbolCache.ContainsKey(canonicalSymbol)) + if (TryParseFutureSymbol(futuresTicker, out var leanFutureSymbol)) { var placeholderSymbolData = new SymbolData { - Symbol = canonicalSymbol, + Symbol = leanFutureSymbol, SecurityCurrency = Currencies.USD, SecurityExchange = market, StartPosition = prevPosition, - EndPosition = currentPosition + EndPosition = currentPosition, + Ticker = columns[columnSymbol] }; - symbolCache.Add(canonicalSymbol, placeholderSymbolData); + if (symbolCache.TryAdd(leanFutureSymbol, placeholderSymbolData)) + { + break; + } + } + + canonicalSymbol = Symbol.Create(underlyingString, SecurityType.Future, market); + + if (symbolCache.TryGetValue(canonicalSymbol, out var symbolData)) + { + symbolData.EndPosition = currentPosition; } else { - symbolCache[canonicalSymbol].EndPosition = currentPosition; + var placeholderSymbolData = new SymbolData + { + Symbol = canonicalSymbol, + SecurityCurrency = Currencies.USD, + SecurityExchange = market, + StartPosition = prevPosition, + EndPosition = currentPosition, + Ticker = columns[columnSymbol] + }; + + symbolCache.Add(canonicalSymbol, placeholderSymbolData); } break; @@ -517,7 +536,7 @@ private IEnumerable LoadSymbolOnDemand(SymbolData placeholder) continue; } - var futuresTicker = columns[columnSymbol].TrimStart(new[] { '@' }); + var futuresTicker = NormalizeFuturesTicker(columns[columnExchange], columns[columnSymbol]); var parsed = SymbolRepresentation.ParseFutureTicker(futuresTicker); var underlyingString = parsed.Underlying; @@ -695,12 +714,6 @@ public Tuple Request(string ticker) } } - public IEnumerable GetBrokerageContractSymbol(Symbol subscribeSymbol) - { - return _symbols.Where(kpv => kpv.Key.SecurityType == SecurityType.Future && kpv.Key.ID.Symbol == subscribeSymbol.ID.Symbol) - .Select(kpv => kpv.Value); - } - /// /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. /// @@ -708,5 +721,39 @@ public void Dispose() { _dataCacheProvider.DisposeSafely(); } + + /// + /// Normalizes the futures ticker by removing exchange-specific electronic contract prefixes. + /// + /// The exchange code (e.g., "NYMEX", "COMEX", "CBOT", "CME"). + /// The raw symbol from the data feed. + /// The normalized symbol with the exchange-specific prefix removed. + /// + /// NYMEX and COMEX adopted the 'Q' prefix, whereas CBOT and CME use '@' to represent electronic contracts. + /// For unknown exchanges, '@' is removed by default to preserve legacy behavior. + /// + private static string NormalizeFuturesTicker(string exchange, string symbol) + { + return exchange switch + { + "COMEX" or "NYMEX" => symbol.TrimStart('Q'), + "CBOT" or "CME" => symbol.TrimStart('@'), + _ => symbol.TrimStart('@') // Legacy fallback: default handling for unknown exchanges + }; + } + + /// + /// Attempts to parse the specified future ticker string into a Lean . + /// + /// The future ticker string to parse. + /// + /// When this method returns, contains the parsed if the parsing succeeded; otherwise, null. + /// + /// true if the parsing succeeded; otherwise, false. + private static bool TryParseFutureSymbol(string futureTicker, out Symbol leanSymbol) + { + leanSymbol = SymbolRepresentation.ParseFutureSymbol(futureTicker); + return leanSymbol != null; + } } } From 722814792a0e6a878780f0727b8d12b37b817008 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 28 Jul 2025 23:04:30 +0300 Subject: [PATCH 03/25] test:feat: DataQueueUniverseProvider --- .../IQFeedDataQueueUniverseProviderTests.cs | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs diff --git a/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs b/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs new file mode 100644 index 0000000..3da7b9e --- /dev/null +++ b/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs @@ -0,0 +1,52 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using System.Linq; +using NUnit.Framework; +using QuantConnect.Interfaces; +using QuantConnect.Securities; +using System.Collections.Generic; + +namespace QuantConnect.Lean.DataSource.IQFeed.Tests; + +[TestFixture] +public class IQFeedDataQueueUniverseProviderTests +{ + private IDataQueueUniverseProvider _iQFeedDataProvider; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + _iQFeedDataProvider = new IQFeedDataProvider(); + } + + private static IEnumerable LookUpSymbolsTestParameters + { + get + { + yield return new TestCaseData(Symbol.CreateFuture(Futures.Energy.NaturalGas, Market.NYMEX, new DateTime(2025, 08, 27))); + yield return new TestCaseData(Symbol.CreateFuture(Futures.Indices.NASDAQ100EMini, Market.CME, new DateTime(2025, 09, 19))); + } + } + + [Test, TestCaseSource(nameof(LookUpSymbolsTestParameters))] + public void LookUpSymbols(Symbol symbol) + { + var symbols = _iQFeedDataProvider.LookupSymbols(symbol, true, default).ToList(); + Assert.IsNotEmpty(symbols); + } +} From 6bf423a5d524469532d839f9a0a95b0adb5a1cb9 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 28 Jul 2025 23:04:58 +0300 Subject: [PATCH 04/25] test:feat: SymbolMapper --- .../IQFeedSymbolRepresentationTests.cs | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/QuantConnect.IQFeed.Tests/IQFeedSymbolRepresentationTests.cs b/QuantConnect.IQFeed.Tests/IQFeedSymbolRepresentationTests.cs index e91e1bd..f7524d5 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedSymbolRepresentationTests.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedSymbolRepresentationTests.cs @@ -15,12 +15,30 @@ using System; using NUnit.Framework; +using QuantConnect.Securities; +using System.Collections.Generic; +using QuantConnect.Configuration; namespace QuantConnect.Lean.DataSource.IQFeed.Tests { - [TestFixture] + [TestFixture, Explicit("Requires locally installed and configured IQFeed client, including login credentials and active data subscription.")] public class IQFeedSymbolRepresentationTests { + private IQFeedDataQueueUniverseProvider _dataQueueUniverseProvider; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + if (OS.IsWindows) + { + // IQConnect is only supported on Windows + var connector = new IQConnect(Config.Get("iqfeed-productName"), "1.0"); + Assert.IsTrue(connector.Launch(), "Failed to launch IQConnect on Windows. Ensure IQFeed is installed and configured properly."); + } + + _dataQueueUniverseProvider = new IQFeedDataQueueUniverseProvider(); + } + [Test] public void ParseOptionIQFeedTicker() { @@ -50,5 +68,34 @@ public void IQFeedTickerRoundTrip(string encodedOption, string underlying, Optio Assert.AreEqual(encodedOption, result); } + + public static IEnumerable GetFutureSymbolsTestCases() + { + // Natural gas futures expire the month previous to the contract month: + // Expiry: August -> Contract month: September (U) + yield return new TestCaseData("QNGU25", Symbol.CreateFuture(Futures.Energy.NaturalGas, Market.NYMEX, new DateTime(2025, 08, 27))); + // Expiry: December 2025 -> Contract month: January (U) 2026 (26) + yield return new TestCaseData("QNGF26", Symbol.CreateFuture(Futures.Energy.NaturalGas, Market.NYMEX, new DateTime(2025, 12, 29))); + + // BrentLastDayFinancial futures expire two months previous to the contract month: + // Expiry: August -> Contract month: October (V) + yield return new TestCaseData("QBZV25", Symbol.CreateFuture(Futures.Energy.BrentLastDayFinancial, Market.NYMEX, new DateTime(2025, 08, 29))); + // Expiry: November 2025 -> Contract month: January (F) 2026 (26) + yield return new TestCaseData("QBZF26", Symbol.CreateFuture(Futures.Energy.BrentLastDayFinancial, Market.NYMEX, new DateTime(2025, 11, 28))); + // Expiry: December 2025 -> Contract month: February (G) 2026 (26) + yield return new TestCaseData("QBZG26", Symbol.CreateFuture(Futures.Energy.BrentLastDayFinancial, Market.NYMEX, new DateTime(2025, 12, 31))); + + yield return new TestCaseData("@NQU25", Symbol.CreateFuture(Futures.Indices.NASDAQ100EMini, Market.CME, new DateTime(2025, 09, 19))); + } + + [TestCaseSource(nameof(GetFutureSymbolsTestCases))] + public void ConvertsFutureSymbolRoundTrip(string brokerageSymbol, Symbol leanSymbol) + { + var actualBrokerageSymbol = _dataQueueUniverseProvider.GetBrokerageSymbol(leanSymbol); + var actualLeanSymbol = _dataQueueUniverseProvider.GetLeanSymbol(brokerageSymbol, default, default); + + Assert.AreEqual(brokerageSymbol, actualBrokerageSymbol); + Assert.AreEqual(leanSymbol, actualLeanSymbol); + } } } From 1ddb33358d267215409e7b43dc0600a5dadc19af Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 29 Jul 2025 17:54:44 +0300 Subject: [PATCH 05/25] fix: FuturesExchanges collection test:feat: add future history requests --- .../IQFeedHistoryProviderTests.cs | 22 +++++++++---- .../IQFeedDataQueueUniverseProvider.cs | 33 ++++++++++++++----- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs b/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs index cac456a..18cfac9 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs @@ -30,8 +30,8 @@ public class IQFeedHistoryProviderTests { private IQFeedDataProvider _historyProvider; - [SetUp] - public void SetUp() + [OneTimeSetUp] + public void OneTimeSetUp() { _historyProvider = new IQFeedDataProvider(); _historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, null, null, null)); @@ -61,13 +61,21 @@ internal static IEnumerable HistoricalTestParameters yield return new TestCaseData(AAPL, Resolution.Hour, TickType.Quote, TimeSpan.FromDays(180), true); yield return new TestCaseData(AAPL, Resolution.Daily, TickType.Quote, TimeSpan.FromDays(365), true); - // TickType.OpenInterest is not maintained - yield return new TestCaseData(AAPL, Resolution.Tick, TickType.OpenInterest, TimeSpan.FromMinutes(5), true); + // Future + var nasdaq100EMini = Symbol.CreateFuture(Futures.Indices.NASDAQ100EMini, Market.CME, new DateTime(2025, 09, 19)); + yield return new TestCaseData(nasdaq100EMini, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(180), false); + yield return new TestCaseData(nasdaq100EMini, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(180), false); + yield return new TestCaseData(nasdaq100EMini, Resolution.Minute, TickType.Trade, TimeSpan.FromDays(10), false); + yield return new TestCaseData(nasdaq100EMini, Resolution.Second, TickType.Trade, TimeSpan.FromMinutes(10), false); + yield return new TestCaseData(nasdaq100EMini, Resolution.Tick, TickType.Trade, TimeSpan.FromMinutes(5), false); + + yield return new TestCaseData(AAPL, Resolution.Tick, TickType.OpenInterest, TimeSpan.FromMinutes(5), true).SetDescription("TickType.OpenInterest is not maintained"); // Not supported Security Types yield return new TestCaseData(Symbol.Create("SPX.XO", SecurityType.Index, Market.CBOE), Resolution.Tick, TickType.Trade, TimeSpan.FromMinutes(5), true); - yield return new TestCaseData(Symbol.CreateFuture("@ESGH24", Market.CME, new DateTime(2024, 3, 21)), Resolution.Tick, TickType.Trade, TimeSpan.FromMinutes(5), true); + var naturalGasAug2025 = Symbol.CreateFuture(Futures.Energy.NaturalGas, Market.NYMEX, new DateTime(2025, 08, 27)); + yield return new TestCaseData(naturalGasAug2025, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(100), false); } } @@ -99,7 +107,7 @@ internal static void AssertTicksHaveAppropriateTickType(Resolution resolution, T case (Resolution.Tick, TickType.Trade): Assert.IsTrue(historyResponse.Any(x => x.Ticks.Any(xx => xx.Value.Count > 0 && xx.Value.Any(t => t.TickType == TickType.Trade)))); break; - }; + } } internal static void AssertHistoricalDataResponse(Resolution resolution, List historyResponse) @@ -126,7 +134,7 @@ internal static void AssertHistoricalDataResponse(Resolution resolution, List FuturesExchanges = new Dictionary { - { "CME", Market.Globex }, + { "CBOT_GBX", Market.CBOT }, + { "CBOTMINI", Market.CBOT }, + { "CFE", Market.CFE }, + { "CME_GBX", Market.Globex }, + { "CMEMINI", Market.CME }, + { "COMEX_GBX", Market.COMEX }, + { "EUREX", Market.EUREX }, + { "ICEEA", Market.ICE }, // ICE Futures Europe - included in both Commodities/Financials packages + { "ICEEC", Market.ICE }, // ICE Futures Europe - Commodities + { "ICEEF", Market.ICE }, // ICE Futures Europe - Financials + { "ICEENDEX", Market.ICE }, // ICE Endex + { "ICEFANG", Market.ICE }, // ICE FANG Futures + { "ICEFC", Market.ICE }, // ICE Futures Canada + { "ICEFU", Market.ICE }, // ICE Futures US + { "NYMEX_GBX", Market.NYMEX }, // Nymex Globex Contracts + { "NYMEXMINI", Market.NYMEX }, // NYMEX Mini Contracts + { "SGX", Market.SGX }, // Singapore International Monetary Exchange + { "CME", Market.CME }, { "NYMEX", Market.NYMEX }, { "CBOT", Market.CBOT }, - { "ICEFU", Market.ICE }, - { "CFE", Market.CFE } }; // futures fundamental data resolver @@ -386,7 +401,7 @@ private IEnumerable LoadSymbols() if (columns[columnSymbol].EndsWith("#") || columns[columnSymbol].EndsWith("#C") || columns[columnSymbol].EndsWith("$$")) continue; - var futuresTicker = NormalizeFuturesTicker(columns[columnExchange], columns[columnSymbol]); + var futuresTicker = NormalizeFuturesTicker(columns[columnListedMarket], columns[columnSymbol]); var parsed = SymbolRepresentation.ParseFutureTicker(futuresTicker); var underlyingString = parsed.Underlying; @@ -410,7 +425,7 @@ private IEnumerable LoadSymbols() } } - var market = GetFutureMarket(underlyingString, columns[columnExchange]); + var market = GetFutureMarket(underlyingString, columns[columnListedMarket]); if (TryParseFutureSymbol(futuresTicker, out var leanFutureSymbol)) { @@ -536,7 +551,7 @@ private IEnumerable LoadSymbolOnDemand(SymbolData placeholder) continue; } - var futuresTicker = NormalizeFuturesTicker(columns[columnExchange], columns[columnSymbol]); + var futuresTicker = NormalizeFuturesTicker(columns[columnListedMarket], columns[columnSymbol]); var parsed = SymbolRepresentation.ParseFutureTicker(futuresTicker); var underlyingString = parsed.Underlying; @@ -549,7 +564,7 @@ private IEnumerable LoadSymbolOnDemand(SymbolData placeholder) continue; } - var market = GetFutureMarket(underlyingString, columns[columnExchange]); + var market = GetFutureMarket(underlyingString, columns[columnListedMarket]); // Futures contracts have different idiosyncratic expiration dates that IQFeed symbol universe file doesn't contain // We request IQFeed explicitly for the exact expiration data of each contract @@ -736,8 +751,8 @@ private static string NormalizeFuturesTicker(string exchange, string symbol) { return exchange switch { - "COMEX" or "NYMEX" => symbol.TrimStart('Q'), - "CBOT" or "CME" => symbol.TrimStart('@'), + "COMEX" or "NYMEX" or "NYMEXMINI" or "NYMEX_GBX" => symbol.TrimStart('Q'), + "CBOT" or "CBOT_GBX" or "CBOTMINI" or "CME" or "CMEMINI" or "COMEX_GBX" => symbol.TrimStart('@'), _ => symbol.TrimStart('@') // Legacy fallback: default handling for unknown exchanges }; } From 1aef2864146c3e58485fcd57db848606966c2ad1 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 29 Jul 2025 20:08:43 +0300 Subject: [PATCH 06/25] feat: implement canonical downloading --- QuantConnect.IQFeed/IQFeedDataDownloader.cs | 150 ++++++++++++++++---- 1 file changed, 125 insertions(+), 25 deletions(-) diff --git a/QuantConnect.IQFeed/IQFeedDataDownloader.cs b/QuantConnect.IQFeed/IQFeedDataDownloader.cs index 0bcb5d0..8d751e9 100644 --- a/QuantConnect.IQFeed/IQFeedDataDownloader.cs +++ b/QuantConnect.IQFeed/IQFeedDataDownloader.cs @@ -14,12 +14,13 @@ * */ +using NodaTime; using QuantConnect.Data; -using QuantConnect.Logging; using QuantConnect.Securities; using QuantConnect.Data.Market; using QuantConnect.Configuration; using IQFeed.CSharpApiClient.Lookup; +using System.Collections.Concurrent; namespace QuantConnect.Lean.DataSource.IQFeed { @@ -34,32 +35,34 @@ public class IQFeedDataDownloader : IDataDownloader private const int NumberOfClients = 8; /// - /// Lazy initialization for the IQFeed file history provider. + /// Provides access to all available symbols corresponding to a canonical symbol using the IQFeed data source. /// - /// - /// This lazy initialization is used to provide deferred creation of the . - /// - private Lazy _fileHistoryProviderLazy; + private readonly IQFeedDataQueueUniverseProvider _dataQueueUniverseProvider; + + /// + /// Provides access to exchange hours and raw data times zones in various markets + /// + private readonly MarketHoursDatabase _marketHoursDatabase; /// /// The file history provider used by the data downloader. /// - protected IQFeedFileHistoryProvider _fileHistoryProvider => _fileHistoryProviderLazy.Value; + protected readonly IQFeedFileHistoryProvider _fileHistoryProvider; /// /// Initializes a new instance of the class. /// public IQFeedDataDownloader() { - _fileHistoryProviderLazy = new Lazy(() => - { - // Create and connect the IQFeed lookup client - var lookupClient = LookupClientFactory.CreateNew(Config.Get("iqfeed-host", "127.0.0.1"), IQSocket.GetPort(PortType.Lookup), NumberOfClients, LookupDefault.Timeout); - // Establish connection with IQFeed Client - lookupClient.Connect(); + _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - return new IQFeedFileHistoryProvider(lookupClient, new IQFeedDataQueueUniverseProvider(), MarketHoursDatabase.FromDataFolder()); - }); + _dataQueueUniverseProvider = new IQFeedDataQueueUniverseProvider(); + + var lookupClient = LookupClientFactory.CreateNew(Config.Get("iqfeed-host", "127.0.0.1"), IQSocket.GetPort(PortType.Lookup), NumberOfClients, LookupDefault.Timeout); + + lookupClient.Connect(); + + _fileHistoryProvider = new IQFeedFileHistoryProvider(lookupClient, _dataQueueUniverseProvider, MarketHoursDatabase.FromDataFolder()); } /// @@ -75,22 +78,119 @@ public IQFeedDataDownloader() var endUtc = dataDownloaderGetParameters.EndUtc; var tickType = dataDownloaderGetParameters.TickType; + var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); + var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); + var dataType = resolution == Resolution.Tick ? typeof(Tick) : typeof(TradeBar); - return _fileHistoryProvider.ProcessHistoryRequests( - new HistoryRequest( + Logging.Log.Trace($"IS Symbol Cannonical? {symbol} = {symbol.IsCanonical()}"); + + if (symbol.IsCanonical()) + { + return GetCanonicalOptionHistory( + symbol, startUtc, endUtc, dataType, - symbol, resolution, - SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), - TimeZones.NewYork, - resolution, - true, - false, - DataNormalizationMode.Adjusted, - tickType)); + exchangeHours, + dataTimeZone, + dataDownloaderGetParameters.TickType); + } + else + { + return _fileHistoryProvider.ProcessHistoryRequests( + new HistoryRequest( + startUtc, + endUtc, + dataType, + symbol, + resolution, + exchangeHours, + dataTimeZone, + resolution, + true, + false, + DataNormalizationMode.Adjusted, + tickType)); + } + } + + /// + /// Retrieves historical data for all available option contracts derived from a given canonical option symbol + /// within the specified date range and resolution. + /// + /// The canonical option used to derive individual contracts. + /// The UTC start time for the historical data request. + /// The UTC end time for the historical data request. + /// The type of data to retrieve (e.g., , , ). + /// The resolution of the historical data (e.g., Minute, Hour, Daily). + /// The exchange hours of the underlying security. + /// The time zone in which the data timestamps are represented. + /// The tick type of data to retrieve (e.g., Trade, Quote). + /// + /// A collection of representing historical option data, or null if no data was found. + /// + private IEnumerable? GetCanonicalOptionHistory(Symbol symbol, DateTime startUtc, DateTime endUtc, Type dataType, + Resolution resolution, SecurityExchangeHours exchangeHours, DateTimeZone dataTimeZone, TickType tickType) + { + var blockingOptionCollection = new BlockingCollection(); + var symbols = GetOptions(symbol, startUtc, endUtc); + + // Symbol can have a lot of Option parameters + Task.Run(() => Parallel.ForEach(symbols, targetSymbol => + { + var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, targetSymbol, resolution, exchangeHours, dataTimeZone, + resolution, true, false, DataNormalizationMode.Raw, tickType); + + var history = _fileHistoryProvider.ProcessHistoryRequests(historyRequest); + + // If history is null, it indicates an incorrect or missing request for historical data, + // so we skip processing for this symbol and move to the next one. + if (history == null) + { + return; + } + + foreach (var data in history) + { + blockingOptionCollection.Add(data); + } + })).ContinueWith(_ => + { + blockingOptionCollection.CompleteAdding(); + }); + + var options = blockingOptionCollection.GetConsumingEnumerable(); + + // Validate if the collection contains at least one successful response from history. + if (!options.Any()) + { + return null; + } + + return options; + } + + /// + /// Retrieves a distinct set of option symbols for the specified underlying symbol + /// within the given date range, limited to tradeable days as defined by the market hours database. + /// + /// The canonical symbol representing the underlying security. + /// The UTC start date of the lookup period. + /// The UTC end date of the lookup period. + /// + /// An enumerable collection of unique option instances + /// that match the specified underlying symbol over the specified date range. + /// + protected virtual IEnumerable GetOptions(Symbol symbol, DateTime startUtc, DateTime endUtc) + { + var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); + + return QuantConnect.Time.EachTradeableDay(exchangeHours, startUtc.Date, endUtc.Date) + .Select(date => _dataQueueUniverseProvider.LookupSymbols(symbol, default, default)) + .SelectMany(x => x) + .Distinct(); } } } From 220efbfc062d875cbfce447f4ebb4699a8d65c22 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 29 Jul 2025 20:09:43 +0300 Subject: [PATCH 07/25] test:feat: download Canonical Future symbols --- .../IQFeedDataDownloaderTest.cs | 41 ++++++++++++++++++- .../IQFeedHistoryProviderTests.cs | 15 ++++--- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs b/QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs index 1bb6604..7c3c309 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs @@ -17,7 +17,9 @@ using System; using System.Linq; using NUnit.Framework; +using QuantConnect.Securities; using System.Collections.Generic; +using QuantConnect.Configuration; namespace QuantConnect.Lean.DataSource.IQFeed.Tests { @@ -26,9 +28,16 @@ public class IQFeedDataDownloaderTest { private IQFeedDataDownloader _downloader; - [SetUp] - public void SetUp() + [OneTimeSetUp] + public void OneTimeSetUp() { + if (OS.IsWindows) + { + // IQConnect is only supported on Windows + var connector = new IQConnect(Config.Get("iqfeed-productName"), "1.0"); + Assert.IsTrue(connector.Launch(), "Failed to launch IQConnect on Windows. Ensure IQFeed is installed and configured properly."); + } + _downloader = new IQFeedDataDownloader(); } @@ -50,5 +59,33 @@ public void DownloadsHistoricalData(Symbol symbol, Resolution resolution, TickTy IQFeedHistoryProviderTests.AssertHistoricalDataResponse(resolution, downloadResponse); } + + private static IEnumerable CanonicalFutureSymbolTestCases + { + get + { + var startDateUtc = new DateTime(2025, 03, 03); + var endDateUtc = new DateTime(2025, 04, 04); + + var naturalGas = Symbol.Create(Futures.Energy.NaturalGas, SecurityType.Future, Market.NYMEX); + yield return new TestCaseData(naturalGas, Resolution.Daily, TickType.Trade, startDateUtc, endDateUtc); + + var nasdaq100EMini = Symbol.Create(Futures.Indices.NASDAQ100EMini, SecurityType.Future, Market.CME); + yield return new TestCaseData(nasdaq100EMini, Resolution.Daily, TickType.Trade, startDateUtc, endDateUtc); + } + } + + + [TestCaseSource(nameof(CanonicalFutureSymbolTestCases))] + public void DownloadCanonicalFutureHistoricalData(Symbol symbol, Resolution resolution, TickType tickType, DateTime startDateUtc, DateTime endDateUtc) + { + var request = IQFeedHistoryProviderTests.CreateHistoryRequest(symbol, resolution, tickType, startDateUtc, endDateUtc); + + var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType); + var downloadResponse = _downloader.Get(parameters)?.ToList(); + + Assert.IsNotNull(downloadResponse); + Assert.IsNotEmpty(downloadResponse); + } } } diff --git a/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs b/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs index 18cfac9..57c4fc0 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs @@ -20,8 +20,8 @@ using QuantConnect.Util; using QuantConnect.Tests; using QuantConnect.Securities; -using System.Collections.Generic; using QuantConnect.Data.Market; +using System.Collections.Generic; namespace QuantConnect.Lean.DataSource.IQFeed.Tests { @@ -140,11 +140,16 @@ internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution re { end = end.Date.AddDays(1); } - var dataType = LeanData.GetDataType(resolution, tickType); - return new HistoryRequest(end.Subtract(period), - end, - dataType, + return CreateHistoryRequest(symbol, resolution, tickType, end.Subtract(period), end); + } + + internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, DateTime startDateTimeUtc, DateTime endDateTimeUtc) + { + return new HistoryRequest( + startDateTimeUtc, + endDateTimeUtc, + LeanData.GetDataType(resolution, tickType), symbol, resolution, SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), From ce97b8557e2fd2853ef590b3a5e60e5b1648adb6 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 30 Jul 2025 21:16:54 +0300 Subject: [PATCH 08/25] refactor: LoadSymbol Future part test:feat: couple of test of IQFeed-symbol-map.json --- .../IQFeedSymbolRepresentationTests.cs | 6 +++ .../IQFeedDataQueueUniverseProvider.cs | 49 ++++++++++--------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/QuantConnect.IQFeed.Tests/IQFeedSymbolRepresentationTests.cs b/QuantConnect.IQFeed.Tests/IQFeedSymbolRepresentationTests.cs index f7524d5..1d2d2c3 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedSymbolRepresentationTests.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedSymbolRepresentationTests.cs @@ -86,6 +86,12 @@ public static IEnumerable GetFutureSymbolsTestCases() yield return new TestCaseData("QBZG26", Symbol.CreateFuture(Futures.Energy.BrentLastDayFinancial, Market.NYMEX, new DateTime(2025, 12, 31))); yield return new TestCaseData("@NQU25", Symbol.CreateFuture(Futures.Indices.NASDAQ100EMini, Market.CME, new DateTime(2025, 09, 19))); + + yield return new TestCaseData("@ADU25", Symbol.CreateFuture(Futures.Currencies.AUD, Market.CME, new DateTime(2025, 09, 15))) + .SetDescription("IQFeed returns 'AD' for Australian Dollar; Lean expects '6A'. Verifies symbol mapping from 'IQFeed-symbol-map.json'."); + + yield return new TestCaseData("@BOU25", Symbol.CreateFuture(Futures.Grains.SoybeanOil, Market.CBOT, new DateTime(2025, 09, 12))) + .SetDescription("Brokerage uses 'BO' for Soybean Oil; Lean maps it as 'ZL'. Validates symbol translation via 'IQFeed-symbol-map.json'."); } [TestCaseSource(nameof(GetFutureSymbolsTestCases))] diff --git a/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs b/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs index 2cc0279..2a76029 100644 --- a/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs +++ b/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs @@ -283,7 +283,7 @@ private IEnumerable LoadSymbols() if (mapExists) { Log.Trace($"{nameof(IQFeedDataQueueUniverseProvider)}.{nameof(LoadSymbols)}: Loading IQFeed futures symbol map file..."); - _iqFeedNameMap = JsonConvert.DeserializeObject>(File.ReadAllText(iqfeedNameMapFullName)); + _iqFeedNameMap = JsonConvert.DeserializeObject>(File.ReadAllText(iqfeedNameMapFullName)) ?? []; } if (!universeExists) @@ -397,37 +397,40 @@ private IEnumerable LoadSymbols() case "FUTURE": - // we are not interested in designated continuous contracts - if (columns[columnSymbol].EndsWith("#") || columns[columnSymbol].EndsWith("#C") || columns[columnSymbol].EndsWith("$$")) + // Exclude non-standard contracts such as: + // - Continuous contracts (e.g., @BO# or @BO#C) + // - Synthetic or spot instruments (e.g., CVUY$$) + // These are not standard, tradable futures and are not supported by Lean + if (columns[columnSymbol].EndsWith('#') + || columns[columnSymbol].EndsWith("#C", StringComparison.InvariantCultureIgnoreCase) + || columns[columnSymbol].EndsWith("$$", StringComparison.InvariantCultureIgnoreCase)) + { continue; + } - var futuresTicker = NormalizeFuturesTicker(columns[columnListedMarket], columns[columnSymbol]); + var normalizedFutureBrokerageSymbol = NormalizeFuturesTicker(columns[columnListedMarket], columns[columnSymbol]); - var parsed = SymbolRepresentation.ParseFutureTicker(futuresTicker); - var underlyingString = parsed.Underlying; + var underlyingBrokerageTicker = SymbolRepresentation.ParseFutureTicker(normalizedFutureBrokerageSymbol).Underlying; - if (_iqFeedNameMap.TryGetValue(underlyingString, out var underlying)) - underlyingString = underlying; - else + if (_iqFeedNameMap.TryGetValue(underlyingBrokerageTicker, out var underlyingLeanTicker)) { - if (!mapExists) + normalizedFutureBrokerageSymbol = normalizedFutureBrokerageSymbol.Replace(underlyingBrokerageTicker, underlyingLeanTicker); + underlyingBrokerageTicker = underlyingLeanTicker; + } + else if (!mapExists && !_iqFeedNameMap.ContainsKey(underlyingBrokerageTicker)) + { + // if map is not created yet, we request this information from IQFeed + var exchangeSymbol = _symbolFundamentalData.Request(columns[columnSymbol]).Item2; + if (!string.IsNullOrEmpty(exchangeSymbol)) { - if (!_iqFeedNameMap.ContainsKey(underlyingString)) - { - // if map is not created yet, we request this information from IQFeed - var exchangeSymbol = _symbolFundamentalData.Request(columns[columnSymbol]).Item2; - if (!string.IsNullOrEmpty(exchangeSymbol)) - { - _iqFeedNameMap[underlyingString] = exchangeSymbol; - underlyingString = exchangeSymbol; - } - } + _iqFeedNameMap[underlyingBrokerageTicker] = exchangeSymbol; + underlyingBrokerageTicker = exchangeSymbol; } } - var market = GetFutureMarket(underlyingString, columns[columnListedMarket]); + var market = GetFutureMarket(underlyingBrokerageTicker, columns[columnListedMarket]); - if (TryParseFutureSymbol(futuresTicker, out var leanFutureSymbol)) + if (TryParseFutureSymbol(normalizedFutureBrokerageSymbol, out var leanFutureSymbol)) { var placeholderSymbolData = new SymbolData { @@ -445,7 +448,7 @@ private IEnumerable LoadSymbols() } } - canonicalSymbol = Symbol.Create(underlyingString, SecurityType.Future, market); + canonicalSymbol = Symbol.Create(underlyingBrokerageTicker, SecurityType.Future, market); if (symbolCache.TryGetValue(canonicalSymbol, out var symbolData)) { From 9488e633efda1e6e5c9512e72523130e745843b1 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 30 Jul 2025 21:48:02 +0300 Subject: [PATCH 09/25] refactor: normalizeFutureSymbol method test:feat: NormalizeFuturesTicker --- .../IQFeedSymbolRepresentationTests.cs | 8 +++++++ .../IQFeedDataQueueUniverseProvider.cs | 24 ++++++++++++------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/QuantConnect.IQFeed.Tests/IQFeedSymbolRepresentationTests.cs b/QuantConnect.IQFeed.Tests/IQFeedSymbolRepresentationTests.cs index 1d2d2c3..1f69f39 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedSymbolRepresentationTests.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedSymbolRepresentationTests.cs @@ -103,5 +103,13 @@ public void ConvertsFutureSymbolRoundTrip(string brokerageSymbol, Symbol leanSym Assert.AreEqual(brokerageSymbol, actualBrokerageSymbol); Assert.AreEqual(leanSymbol, actualLeanSymbol); } + + [TestCase("NYMEX_GBX", "QQA", "QA")] + [TestCase("NYMEX_GBX", "QQAN25", "QAN25")] + public void NormalizeFuturesTickerRemovesFirstQForNymexGbxExchange(string exchange, string brokerageSymbol, string expectedNormalizedFutureSymbol) + { + var actualNormalizedFutureSymbol = IQFeedDataQueueUniverseProvider.NormalizeFuturesTicker(exchange, brokerageSymbol); + Assert.AreEqual(expectedNormalizedFutureSymbol, actualNormalizedFutureSymbol); + } } } diff --git a/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs b/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs index 2a76029..1dd719f 100644 --- a/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs +++ b/QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs @@ -22,9 +22,12 @@ using QuantConnect.Interfaces; using QuantConnect.Brokerages; using QuantConnect.Securities; +using System.Runtime.CompilerServices; using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.DataFeeds.Transport; +[assembly: InternalsVisibleTo("QuantConnect.Lean.DataSource.IQFeed.Tests")] + namespace QuantConnect.Lean.DataSource.IQFeed { /// @@ -747,17 +750,22 @@ public void Dispose() /// The raw symbol from the data feed. /// The normalized symbol with the exchange-specific prefix removed. /// - /// NYMEX and COMEX adopted the 'Q' prefix, whereas CBOT and CME use '@' to represent electronic contracts. - /// For unknown exchanges, '@' is removed by default to preserve legacy behavior. + /// NYMEX, COMEX, NYMEXMINI, and NYMEX_GBX adopted the 'Q' prefix, which is removed explicitly. + /// Exchanges such as CBOT, CBOT_GBX, CBOTMINI, CME, CMEMINI, and COMEX_GBX typically use the '@' + /// prefix to denote electronic contracts. This '@' is removed by default in the fallback case. /// - private static string NormalizeFuturesTicker(string exchange, string symbol) + internal static string NormalizeFuturesTicker(string exchange, string symbol) { - return exchange switch + switch (exchange) { - "COMEX" or "NYMEX" or "NYMEXMINI" or "NYMEX_GBX" => symbol.TrimStart('Q'), - "CBOT" or "CBOT_GBX" or "CBOTMINI" or "CME" or "CMEMINI" or "COMEX_GBX" => symbol.TrimStart('@'), - _ => symbol.TrimStart('@') // Legacy fallback: default handling for unknown exchanges - }; + case "COMEX": + case "NYMEX": + case "NYMEXMINI": + case "NYMEX_GBX": + return symbol[1..]; + default: + return symbol[0] == '@' ? symbol[1..] : symbol; + } } /// From dad1b87af7fc7c37ee4bfb7232b39deb05b2355c Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 30 Jul 2025 21:58:31 +0300 Subject: [PATCH 10/25] test:refactor: download future canonical with multiple symbols --- QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs b/QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs index 7c3c309..b6ca90d 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs @@ -79,13 +79,15 @@ private static IEnumerable CanonicalFutureSymbolTestCases [TestCaseSource(nameof(CanonicalFutureSymbolTestCases))] public void DownloadCanonicalFutureHistoricalData(Symbol symbol, Resolution resolution, TickType tickType, DateTime startDateUtc, DateTime endDateUtc) { - var request = IQFeedHistoryProviderTests.CreateHistoryRequest(symbol, resolution, tickType, startDateUtc, endDateUtc); - - var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType); + var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateUtc, endDateUtc, tickType); var downloadResponse = _downloader.Get(parameters)?.ToList(); Assert.IsNotNull(downloadResponse); Assert.IsNotEmpty(downloadResponse); + + var uniqueFutureSymbols = downloadResponse.Select(x => x.Symbol).Distinct().ToList(); + + Assert.That(uniqueFutureSymbols.Count, Is.GreaterThan(1), $"Expected more than 1 unique future symbol, but got {uniqueFutureSymbols.Count}: {string.Join(", ", uniqueFutureSymbols)}"); } } } From 804909ab4d0351a1704e2d75f92cd7d010bdf14e Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 30 Jul 2025 22:03:29 +0300 Subject: [PATCH 11/25] test:feat; test LookupSymbols with Equity --- .../IQFeedDataQueueUniverseProviderTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs b/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs index 3da7b9e..03806d2 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs @@ -17,6 +17,7 @@ using System; using System.Linq; using NUnit.Framework; +using QuantConnect.Tests; using QuantConnect.Interfaces; using QuantConnect.Securities; using System.Collections.Generic; @@ -40,6 +41,8 @@ private static IEnumerable LookUpSymbolsTestParameters { yield return new TestCaseData(Symbol.CreateFuture(Futures.Energy.NaturalGas, Market.NYMEX, new DateTime(2025, 08, 27))); yield return new TestCaseData(Symbol.CreateFuture(Futures.Indices.NASDAQ100EMini, Market.CME, new DateTime(2025, 09, 19))); + yield return new TestCaseData(Symbol.CreateCanonicalOption(Symbols.AAPL)); + yield return new TestCaseData(Symbol.CreateCanonicalOption(Symbols.SPY)); } } From 9b80b97dc11c4e7659dffa9c1c4f9c922865929d Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 30 Jul 2025 22:07:56 +0300 Subject: [PATCH 12/25] test:feat: unsupported Canonical Option History request --- QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs b/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs index 57c4fc0..132d3fc 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs @@ -76,6 +76,8 @@ internal static IEnumerable HistoricalTestParameters var naturalGasAug2025 = Symbol.CreateFuture(Futures.Energy.NaturalGas, Market.NYMEX, new DateTime(2025, 08, 27)); yield return new TestCaseData(naturalGasAug2025, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(100), false); + + yield return new TestCaseData(Symbol.CreateCanonicalOption(Symbols.AAPL), default, default, default, true).SetDescription("Canonical symbol requests are unsupported and return null."); } } From fca4b41c1076d9a4402955a087e325ce12e8b6ad Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 30 Jul 2025 22:12:51 +0300 Subject: [PATCH 13/25] rollback: lazy object in Downloader class --- QuantConnect.IQFeed/IQFeedDataDownloader.cs | 23 +++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/QuantConnect.IQFeed/IQFeedDataDownloader.cs b/QuantConnect.IQFeed/IQFeedDataDownloader.cs index 8d751e9..fa1a351 100644 --- a/QuantConnect.IQFeed/IQFeedDataDownloader.cs +++ b/QuantConnect.IQFeed/IQFeedDataDownloader.cs @@ -44,10 +44,18 @@ public class IQFeedDataDownloader : IDataDownloader /// private readonly MarketHoursDatabase _marketHoursDatabase; + /// + /// Lazy initialization for the IQFeed file history provider. + /// + /// + /// This lazy initialization is used to provide deferred creation of the . + /// + private Lazy _fileHistoryProviderLazy; + /// /// The file history provider used by the data downloader. /// - protected readonly IQFeedFileHistoryProvider _fileHistoryProvider; + protected IQFeedFileHistoryProvider _fileHistoryProvider => _fileHistoryProviderLazy.Value; /// /// Initializes a new instance of the class. @@ -55,14 +63,17 @@ public class IQFeedDataDownloader : IDataDownloader public IQFeedDataDownloader() { _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - _dataQueueUniverseProvider = new IQFeedDataQueueUniverseProvider(); - var lookupClient = LookupClientFactory.CreateNew(Config.Get("iqfeed-host", "127.0.0.1"), IQSocket.GetPort(PortType.Lookup), NumberOfClients, LookupDefault.Timeout); - - lookupClient.Connect(); + _fileHistoryProviderLazy = new Lazy(() => + { + // Create and connect the IQFeed lookup client + var lookupClient = LookupClientFactory.CreateNew(Config.Get("iqfeed-host", "127.0.0.1"), IQSocket.GetPort(PortType.Lookup), NumberOfClients, LookupDefault.Timeout); + // Establish connection with IQFeed Client + lookupClient.Connect(); - _fileHistoryProvider = new IQFeedFileHistoryProvider(lookupClient, _dataQueueUniverseProvider, MarketHoursDatabase.FromDataFolder()); + return new IQFeedFileHistoryProvider(lookupClient, _dataQueueUniverseProvider, MarketHoursDatabase.FromDataFolder()); + }); } /// From 0b12af3e696fe20ebb883d0d524988bd402281de Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 30 Jul 2025 22:14:32 +0300 Subject: [PATCH 14/25] remove: extra log trace in Downloader --- QuantConnect.IQFeed/IQFeedDataDownloader.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/QuantConnect.IQFeed/IQFeedDataDownloader.cs b/QuantConnect.IQFeed/IQFeedDataDownloader.cs index fa1a351..8aebc8c 100644 --- a/QuantConnect.IQFeed/IQFeedDataDownloader.cs +++ b/QuantConnect.IQFeed/IQFeedDataDownloader.cs @@ -94,8 +94,6 @@ public IQFeedDataDownloader() var dataType = resolution == Resolution.Tick ? typeof(Tick) : typeof(TradeBar); - Logging.Log.Trace($"IS Symbol Cannonical? {symbol} = {symbol.IsCanonical()}"); - if (symbol.IsCanonical()) { return GetCanonicalOptionHistory( From d505376c6e4cc1835be91304894473b14bab41a5 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 30 Jul 2025 22:21:51 +0300 Subject: [PATCH 15/25] refactor: naming/description in Downloader --- QuantConnect.IQFeed/IQFeedDataDownloader.cs | 49 +++++++++++---------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/QuantConnect.IQFeed/IQFeedDataDownloader.cs b/QuantConnect.IQFeed/IQFeedDataDownloader.cs index 8aebc8c..1e799b6 100644 --- a/QuantConnect.IQFeed/IQFeedDataDownloader.cs +++ b/QuantConnect.IQFeed/IQFeedDataDownloader.cs @@ -96,7 +96,7 @@ public IQFeedDataDownloader() if (symbol.IsCanonical()) { - return GetCanonicalOptionHistory( + return GetCanonicalSymbolHistory( symbol, startUtc, endUtc, @@ -126,25 +126,26 @@ public IQFeedDataDownloader() } /// - /// Retrieves historical data for all available option contracts derived from a given canonical option symbol - /// within the specified date range and resolution. + /// Retrieves historical data for all individual tradeable contracts derived from a given canonical symbol + /// (such as options, futures, or other security types) within the specified date range, resolution, and tick type. /// - /// The canonical option used to derive individual contracts. - /// The UTC start time for the historical data request. - /// The UTC end time for the historical data request. - /// The type of data to retrieve (e.g., , , ). + /// The canonical representing the underlying security group (option chain, future chain, etc.). + /// The UTC start time of the historical data request. + /// The UTC end time of the historical data request. + /// The type of data to retrieve, such as , , or . /// The resolution of the historical data (e.g., Minute, Hour, Daily). - /// The exchange hours of the underlying security. - /// The time zone in which the data timestamps are represented. - /// The tick type of data to retrieve (e.g., Trade, Quote). + /// The exchange hours for the underlying security to correctly filter trading times. + /// The time zone for the timestamps in the returned data. + /// The tick type to retrieve, such as Trade or Quote ticks. /// - /// A collection of representing historical option data, or null if no data was found. + /// An enumerable collection of instances representing historical data for all tradeable contracts + /// derived from the canonical symbol within the requested date range. Returns null if no data was found. /// - private IEnumerable? GetCanonicalOptionHistory(Symbol symbol, DateTime startUtc, DateTime endUtc, Type dataType, + private IEnumerable? GetCanonicalSymbolHistory(Symbol symbol, DateTime startUtc, DateTime endUtc, Type dataType, Resolution resolution, SecurityExchangeHours exchangeHours, DateTimeZone dataTimeZone, TickType tickType) { - var blockingOptionCollection = new BlockingCollection(); - var symbols = GetOptions(symbol, startUtc, endUtc); + var blockingCollection = new BlockingCollection(); + var symbols = GetCanonicalSymbolChain(symbol, startUtc, endUtc); // Symbol can have a lot of Option parameters Task.Run(() => Parallel.ForEach(symbols, targetSymbol => @@ -163,14 +164,14 @@ public IQFeedDataDownloader() foreach (var data in history) { - blockingOptionCollection.Add(data); + blockingCollection.Add(data); } })).ContinueWith(_ => { - blockingOptionCollection.CompleteAdding(); + blockingCollection.CompleteAdding(); }); - var options = blockingOptionCollection.GetConsumingEnumerable(); + var options = blockingCollection.GetConsumingEnumerable(); // Validate if the collection contains at least one successful response from history. if (!options.Any()) @@ -182,17 +183,19 @@ public IQFeedDataDownloader() } /// - /// Retrieves a distinct set of option symbols for the specified underlying symbol - /// within the given date range, limited to tradeable days as defined by the market hours database. + /// Retrieves a distinct set of tradeable option symbols for the specified underlying security + /// within the given date range, filtered to trading days as defined by the market hours database. + /// This includes all option types such as equity options, futures options, index options, and more, + /// that are available for trading on those dates. /// - /// The canonical symbol representing the underlying security. + /// The canonical symbol representing the underlying security (e.g., equity, future, or index). /// The UTC start date of the lookup period. /// The UTC end date of the lookup period. /// - /// An enumerable collection of unique option instances - /// that match the specified underlying symbol over the specified date range. + /// An enumerable collection of unique instances representing tradeable options + /// for the specified underlying symbol over the specified date range. /// - protected virtual IEnumerable GetOptions(Symbol symbol, DateTime startUtc, DateTime endUtc) + protected virtual IEnumerable GetCanonicalSymbolChain(Symbol symbol, DateTime startUtc, DateTime endUtc) { var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); From 1d6bc40c850c9cb54bd1b7c9db6641807603b47a Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 30 Jul 2025 23:44:22 +0300 Subject: [PATCH 16/25] test:fix: use canonical future instead expiry ones --- .../IQFeedDataQueueUniverseProviderTests.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs b/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs index 03806d2..e1f9510 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedDataQueueUniverseProviderTests.cs @@ -39,8 +39,8 @@ private static IEnumerable LookUpSymbolsTestParameters { get { - yield return new TestCaseData(Symbol.CreateFuture(Futures.Energy.NaturalGas, Market.NYMEX, new DateTime(2025, 08, 27))); - yield return new TestCaseData(Symbol.CreateFuture(Futures.Indices.NASDAQ100EMini, Market.CME, new DateTime(2025, 09, 19))); + yield return new TestCaseData(Symbol.Create(Futures.Energy.NaturalGas, SecurityType.Future, Market.NYMEX)); + yield return new TestCaseData(Symbol.Create(Futures.Indices.NASDAQ100EMini, SecurityType.Future, Market.CME)); yield return new TestCaseData(Symbol.CreateCanonicalOption(Symbols.AAPL)); yield return new TestCaseData(Symbol.CreateCanonicalOption(Symbols.SPY)); } @@ -51,5 +51,7 @@ public void LookUpSymbols(Symbol symbol) { var symbols = _iQFeedDataProvider.LookupSymbols(symbol, true, default).ToList(); Assert.IsNotEmpty(symbols); + Assert.Greater(symbols.Count, 1); + Assert.AreEqual(symbols.Count, symbols.Distinct().Count()); } } From e14b217c863c12f078de42d3a289a5a1d9508c7f Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 31 Jul 2025 15:43:28 +0300 Subject: [PATCH 17/25] rename: variable to more fit in Data Downloader --- QuantConnect.IQFeed/IQFeedDataDownloader.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/QuantConnect.IQFeed/IQFeedDataDownloader.cs b/QuantConnect.IQFeed/IQFeedDataDownloader.cs index 1e799b6..fed6e07 100644 --- a/QuantConnect.IQFeed/IQFeedDataDownloader.cs +++ b/QuantConnect.IQFeed/IQFeedDataDownloader.cs @@ -171,15 +171,15 @@ public IQFeedDataDownloader() blockingCollection.CompleteAdding(); }); - var options = blockingCollection.GetConsumingEnumerable(); + var historyResponses = blockingCollection.GetConsumingEnumerable(); // Validate if the collection contains at least one successful response from history. - if (!options.Any()) + if (!historyResponses.Any()) { return null; } - return options; + return historyResponses; } /// From 4505cdf81256f8d24ddf3756f58443990be47e1a Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 31 Jul 2025 22:03:57 +0300 Subject: [PATCH 18/25] refactor: handle history request --- .../IQFeedAPI/IQLookupHistorySymbolClient.cs | 45 ++++++-------- QuantConnect.IQFeed/IQFeedDataProvider.cs | 45 ++++++++------ .../Models/HistoryRequestContext.cs | 62 +++++++++++++++++++ 3 files changed, 107 insertions(+), 45 deletions(-) create mode 100644 QuantConnect.IQFeed/Models/HistoryRequestContext.cs diff --git a/QuantConnect.IQFeed/IQFeedAPI/IQLookupHistorySymbolClient.cs b/QuantConnect.IQFeed/IQFeedAPI/IQLookupHistorySymbolClient.cs index 5535d8e..87165cc 100644 --- a/QuantConnect.IQFeed/IQFeedAPI/IQLookupHistorySymbolClient.cs +++ b/QuantConnect.IQFeed/IQFeedAPI/IQLookupHistorySymbolClient.cs @@ -65,8 +65,16 @@ public LookupTickEventArgs(string requestId, string line) : public class LookupIntervalEventArgs : LookupEventArgs { - public LookupIntervalEventArgs(string requestId, string line) : - base(requestId, LookupType.REQ_HST_INT, LookupSequence.MessageDetail) + public DateTime DateTimeStamp { get; } + public decimal High { get; } + public decimal Low { get; } + public decimal Open { get; } + public decimal Close { get; } + public int TotalVolume { get; } + public int PeriodVolume { get; } + + public LookupIntervalEventArgs(string requestId, string line) + : base(requestId, LookupType.REQ_HST_INT, LookupSequence.MessageDetail) { var fields = line.Split(','); if (fields.Length < 8) @@ -74,32 +82,15 @@ public LookupIntervalEventArgs(string requestId, string line) : Log.Error("LookupIntervalEventArgs.ctor(): " + line); return; } - if (!DateTime.TryParseExact(fields[1], "yyyy-MM-dd HH:mm:ss", _enUS, DateTimeStyles.None, out _dateTimeStamp)) _dateTimeStamp = DateTime.MinValue; - if (!double.TryParse(fields[2], out _high)) _high = 0; - if (!double.TryParse(fields[3], out _low)) _low = 0; - if (!double.TryParse(fields[4], out _open)) _open = 0; - if (!double.TryParse(fields[5], out _close)) _close = 0; - if (!int.TryParse(fields[6], out _totalVolume)) _totalVolume = 0; - if (!int.TryParse(fields[7], out _periodVolume)) _periodVolume = 0; - } - public DateTime DateTimeStamp { get { return _dateTimeStamp; } } - public double High { get { return _high; } } - public double Low { get { return _low; } } - public double Open { get { return _open; } } - public double Close { get { return _close; } } - public int TotalVolume { get { return _totalVolume; } } - public int PeriodVolume { get { return _periodVolume; } } - #region private - private DateTime _dateTimeStamp; - private double _high; - private double _low; - private double _open; - private double _close; - private int _totalVolume; - private int _periodVolume; - private CultureInfo _enUS = new CultureInfo("en-US"); - #endregion + DateTimeStamp = DateTime.TryParseExact(fields[1], "yyyy-MM-dd HH:mm:ss", new CultureInfo("en-US"), DateTimeStyles.None, out var dateTimeStamp) ? dateTimeStamp : default; + High = decimal.TryParse(fields[2], out var h) ? h : 0m; + Low = decimal.TryParse(fields[3], out var l) ? l : 0m; + Open = decimal.TryParse(fields[4], out var o) ? o : 0m; + Close = decimal.TryParse(fields[5], out var c) ? c : 0m; + TotalVolume = int.TryParse(fields[6], out var t) ? t : 0; + PeriodVolume = int.TryParse(fields[7], out var p) ? p : 0; + } } public class LookupDayWeekMonthEventArgs : LookupEventArgs diff --git a/QuantConnect.IQFeed/IQFeedDataProvider.cs b/QuantConnect.IQFeed/IQFeedDataProvider.cs index 417601a..e0d4484 100644 --- a/QuantConnect.IQFeed/IQFeedDataProvider.cs +++ b/QuantConnect.IQFeed/IQFeedDataProvider.cs @@ -33,6 +33,7 @@ using System.Security.Cryptography; using System.Collections.Concurrent; using System.Net.NetworkInformation; +using QuantConnect.Lean.DataSource.IQFeed.Models; using HistoryRequest = QuantConnect.Data.HistoryRequest; namespace QuantConnect.Lean.DataSource.IQFeed @@ -757,10 +758,15 @@ private void OnLevel1UnknownEvent(object sender, Level1TextLineEventArgs e) public class HistoryPort : IQLookupHistorySymbolClient { private bool _inProgress; - private ConcurrentDictionary _requestDataByRequestId; + private readonly ConcurrentDictionary _requestDataByRequestId = []; private ConcurrentDictionary> _currentRequest; private readonly IQFeedDataQueueUniverseProvider _symbolUniverse; + /// + /// Represents the time zone used by IQFeed, which returns time in the New York (EST) Time Zone with daylight savings time. + /// + private static DateTimeZone TimeZoneIQFeed = TimeZones.NewYork; + /// /// Indicates whether the warning for invalid has been fired. /// @@ -783,7 +789,6 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse) : base(IQFeedDefault.BufferSize) { _symbolUniverse = symbolUniverse; - _requestDataByRequestId = new ConcurrentDictionary(); _currentRequest = new ConcurrentDictionary>(); } @@ -839,8 +844,8 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo _inProgress = true; var ticker = _symbolUniverse.GetBrokerageSymbol(request.Symbol); - var start = request.StartTimeUtc.ConvertFromUtc(TimeZones.NewYork); - DateTime? end = request.EndTimeUtc.ConvertFromUtc(TimeZones.NewYork); + var start = request.StartTimeUtc.ConvertFromUtc(TimeZoneIQFeed); + DateTime? end = request.EndTimeUtc.ConvertFromUtc(TimeZoneIQFeed); var exchangeTz = request.ExchangeHours.TimeZone; // if we're within a minute of now, don't set the end time if (request.EndTimeUtc >= DateTime.UtcNow.AddMinutes(-1)) @@ -872,7 +877,7 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo break; } - _requestDataByRequestId[reqid] = request; + _requestDataByRequestId[reqid] = new(request.Symbol, request.TickType, request.Resolution); while (_inProgress) { @@ -939,8 +944,7 @@ protected override void OnLookupEvent(LookupEventArgs e) _currentRequest.AddOrUpdate(e.Id, new List()); break; case LookupSequence.MessageDetail: - List current; - if (_currentRequest.TryGetValue(e.Id, out current)) + if (_currentRequest.TryGetValue(e.Id, out var current)) { HandleMessageDetail(e, current); } @@ -965,8 +969,7 @@ protected override void OnLookupEvent(LookupEventArgs e) /// Current list of BaseData object private void HandleMessageDetail(LookupEventArgs e, List current) { - var requestData = _requestDataByRequestId[e.Id]; - var data = GetData(e, requestData); + var data = GetData(e, _requestDataByRequestId[e.Id]); if (data != null && data.Time != DateTime.MinValue) { current.Add(data); @@ -979,7 +982,7 @@ private void HandleMessageDetail(LookupEventArgs e, List current) /// Received data /// Request information /// BaseData object - private BaseData GetData(LookupEventArgs e, HistoryRequest requestData) + private BaseData GetData(LookupEventArgs e, HistoryRequestContext requestData) { var isEquity = requestData.Symbol.SecurityType == SecurityType.Equity; try @@ -988,7 +991,7 @@ private BaseData GetData(LookupEventArgs e, HistoryRequest requestData) { case LookupType.REQ_HST_TCK: var t = (LookupTickEventArgs)e; - var time = isEquity ? t.DateTimeStamp : t.DateTimeStamp.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard); + var time = isEquity ? t.DateTimeStamp : t.DateTimeStamp.ConvertTo(TimeZoneIQFeed, TimeZones.EasternStandard); switch (requestData.TickType) { case TickType.Trade: @@ -1017,17 +1020,23 @@ private BaseData GetData(LookupEventArgs e, HistoryRequest requestData) throw new NotImplementedException($"The TickType '{requestData.TickType}' is not supported in the {nameof(GetData)} method. Please implement the necessary logic for handling this TickType."); } case LookupType.REQ_HST_INT: - var i = (LookupIntervalEventArgs)e; - if (i.DateTimeStamp == DateTime.MinValue) return null; - var istartTime = i.DateTimeStamp - requestData.Resolution.ToTimeSpan(); - if (!isEquity) istartTime = istartTime.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard); - return new TradeBar(istartTime, requestData.Symbol, (decimal)i.Open, (decimal)i.High, (decimal)i.Low, (decimal)i.Close, i.PeriodVolume, requestData.Resolution.ToTimeSpan()); + if (e is LookupIntervalEventArgs i && i.DateTimeStamp != default) + { + var startTime = i.DateTimeStamp - requestData.Resolution; + if (!isEquity) + { + startTime = startTime.ConvertTo(TimeZoneIQFeed, requestData.SymbolDateTimeZone); + } + + return new TradeBar(startTime, requestData.Symbol, i.Open, i.High, i.Low, i.Close, i.PeriodVolume, requestData.Resolution); + } + return null; case LookupType.REQ_HST_DWM: var d = (LookupDayWeekMonthEventArgs)e; if (d.DateTimeStamp == DateTime.MinValue) return null; var dstartTime = d.DateTimeStamp.Date; - if (!isEquity) dstartTime = dstartTime.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard); - return new TradeBar(dstartTime, requestData.Symbol, (decimal)d.Open, (decimal)d.High, (decimal)d.Low, (decimal)d.Close, d.PeriodVolume, requestData.Resolution.ToTimeSpan()); + if (!isEquity) dstartTime = dstartTime.ConvertTo(TimeZoneIQFeed, TimeZones.EasternStandard); + return new TradeBar(dstartTime, requestData.Symbol, (decimal)d.Open, (decimal)d.High, (decimal)d.Low, (decimal)d.Close, d.PeriodVolume, requestData.Resolution); // we don't need to handle these other types case LookupType.REQ_SYM_SYM: diff --git a/QuantConnect.IQFeed/Models/HistoryRequestContext.cs b/QuantConnect.IQFeed/Models/HistoryRequestContext.cs new file mode 100644 index 0000000..131cfb3 --- /dev/null +++ b/QuantConnect.IQFeed/Models/HistoryRequestContext.cs @@ -0,0 +1,62 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using NodaTime; +using QuantConnect.Securities; + +namespace QuantConnect.Lean.DataSource.IQFeed.Models; + +/// +/// Represents a subset of parameters extracted from a history request, +/// providing context for historical data queries related to a specific symbol. +/// +public class HistoryRequestContext +{ + /// + /// Gets the symbol for which historical data is requested. + /// + public Symbol Symbol { get; } + + /// + /// Gets the type of tick data requested (e.g., trade, quote). + /// + public TickType TickType { get; } + + /// + /// Gets the resolution of the historical data request as a . + /// + public TimeSpan Resolution { get; } + + /// + /// Gets the time zone associated with the symbol's exchange. + /// + public DateTimeZone SymbolDateTimeZone { get; } + + /// + /// Initializes a new instance of the class, + /// setting symbol, tick type, resolution, and determining the symbol's exchange time zone. + /// + /// The symbol representing the financial instrument. + /// The tick data type (trade, quote, etc.). + /// The resolution of the requested historical data. + public HistoryRequestContext(Symbol symbol, TickType tickType, Resolution resolution) + { + Symbol = symbol; + TickType = tickType; + Resolution = resolution.ToTimeSpan(); + SymbolDateTimeZone = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; + } +} From a2d6d96c19000591a846115bf9b12d6c26c570db Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 31 Jul 2025 22:10:00 +0300 Subject: [PATCH 19/25] remove: convert time from file --- .../IQFeedFileHistoryProvider.cs | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/QuantConnect.IQFeed/IQFeedFileHistoryProvider.cs b/QuantConnect.IQFeed/IQFeedFileHistoryProvider.cs index 0a91f53..06c8b8b 100644 --- a/QuantConnect.IQFeed/IQFeedFileHistoryProvider.cs +++ b/QuantConnect.IQFeed/IQFeedFileHistoryProvider.cs @@ -156,11 +156,11 @@ private IEnumerable GetDataFromFile(HistoryRequest request, string tic var tickFunc = request.TickType == TickType.Trade ? new Func(CreateTradeTick) : CreateQuoteTick; if (_filesByRequestKeyCache.TryRemove(requestKey, out filename)) - return GetDataFromTickMessages(filename, request, tickFunc, true); + return GetDataFromTickMessages(filename, request.Symbol, tickFunc, true); filename = _lookupClient.Historical.File.GetHistoryTickTimeframeAsync(ticker, startDate, endDate, dataDirection: DataDirection.Oldest).SynchronouslyAwaitTaskResult(); _filesByRequestKeyCache.AddOrUpdate(requestKey, filename); - return GetDataFromTickMessages(filename, request, tickFunc, false); + return GetDataFromTickMessages(filename, request.Symbol, tickFunc, false); case Resolution.Daily: filename = _lookupClient.Historical.File.GetHistoryDailyTimeframeAsync(ticker, startDate, endDate, dataDirection: DataDirection.Oldest).SynchronouslyAwaitTaskResult(); @@ -181,24 +181,23 @@ private IEnumerable GetDataFromFile(HistoryRequest request, string tic } /// - /// Stream IQFeed TickMessages from disk to Lean Tick + /// Streams IQFeed data from a file on disk, + /// converting each valid tick message into Lean's ticks using the provided conversion function. /// - /// - /// - /// - /// - /// Converted Tick - private IEnumerable GetDataFromTickMessages(string filename, HistoryRequest request, Func tickFunc, bool delete) + /// The path to the file containing IQFeed tick messages. + /// The symbol associated with the ticks being processed. + /// A function to convert each into a Lean object. + /// The function receives the tick timestamp, symbol, and original tick message. + /// If true, deletes the source file after processing is complete. + /// An enumerable sequence of ticks converted from the IQFeed tick messages. + private static IEnumerable GetDataFromTickMessages(string filename, Symbol symbol, Func tickFunc, bool delete) { - var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(request.Symbol.ID.Market, request.Symbol, request.Symbol.SecurityType); - // We need to discard ticks which are not impacting the price, i.e those having BasisForLast = O // To get a better understanding how IQFeed is resampling ticks, have a look to this algorithm: // https://github.com/mathpaquette/IQFeed.CSharpApiClient/blob/1b33250e057dfd6cd77e5ee35fa16aebfc8fbe79/src/IQFeed.CSharpApiClient.Extensions/Lookup/Historical/Resample/TickMessageExtensions.cs#L41 foreach (var tick in TickMessage.ParseFromFile(filename).Where(t => t.BasisForLast != 'O')) { - var timestamp = tick.Timestamp.ConvertTo(TimeZones.NewYork, dataTimeZone); - yield return tickFunc(timestamp, request.Symbol, tick); + yield return tickFunc(tick.Timestamp, symbol, tick); } if (delete) From b5e0557a5e27a6cbdc65409aa0200ffc98ca05da Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 1 Aug 2025 16:36:43 +0300 Subject: [PATCH 20/25] refactor: handle history data responses remove: extra class HistoryRequestContext (keep reusing) --- QuantConnect.IQFeed/IQFeedDataProvider.cs | 107 ++++++++++-------- .../Models/HistoryRequestContext.cs | 62 ---------- 2 files changed, 62 insertions(+), 107 deletions(-) delete mode 100644 QuantConnect.IQFeed/Models/HistoryRequestContext.cs diff --git a/QuantConnect.IQFeed/IQFeedDataProvider.cs b/QuantConnect.IQFeed/IQFeedDataProvider.cs index e0d4484..9733401 100644 --- a/QuantConnect.IQFeed/IQFeedDataProvider.cs +++ b/QuantConnect.IQFeed/IQFeedDataProvider.cs @@ -33,7 +33,6 @@ using System.Security.Cryptography; using System.Collections.Concurrent; using System.Net.NetworkInformation; -using QuantConnect.Lean.DataSource.IQFeed.Models; using HistoryRequest = QuantConnect.Data.HistoryRequest; namespace QuantConnect.Lean.DataSource.IQFeed @@ -758,7 +757,7 @@ private void OnLevel1UnknownEvent(object sender, Level1TextLineEventArgs e) public class HistoryPort : IQLookupHistorySymbolClient { private bool _inProgress; - private readonly ConcurrentDictionary _requestDataByRequestId = []; + private readonly ConcurrentDictionary _requestDataByRequestId = []; private ConcurrentDictionary> _currentRequest; private readonly IQFeedDataQueueUniverseProvider _symbolUniverse; @@ -877,7 +876,7 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo break; } - _requestDataByRequestId[reqid] = new(request.Symbol, request.TickType, request.Resolution); + _requestDataByRequestId[reqid] = request; while (_inProgress) { @@ -982,62 +981,60 @@ private void HandleMessageDetail(LookupEventArgs e, List current) /// Received data /// Request information /// BaseData object - private BaseData GetData(LookupEventArgs e, HistoryRequestContext requestData) + private BaseData GetData(LookupEventArgs e, HistoryRequest requestData) { - var isEquity = requestData.Symbol.SecurityType == SecurityType.Equity; try { switch (e.Type) { case LookupType.REQ_HST_TCK: - var t = (LookupTickEventArgs)e; - var time = isEquity ? t.DateTimeStamp : t.DateTimeStamp.ConvertTo(TimeZoneIQFeed, TimeZones.EasternStandard); - switch (requestData.TickType) + if (e is LookupTickEventArgs t) { - case TickType.Trade: - return new Tick() - { - Time = time, - Value = (decimal)t.Last, - DataType = MarketDataType.Tick, - Symbol = requestData.Symbol, - TickType = TickType.Trade, - Quantity = t.LastSize, - }; - case TickType.Quote: - return new Tick() - { - Time = time, - DataType = MarketDataType.Tick, - Symbol = requestData.Symbol, - TickType = TickType.Quote, - AskPrice = (decimal)t.Ask, - //AskSize = askSize, - BidPrice = (decimal)t.Bid, - //BidSize = bidSize, - }; - default: - throw new NotImplementedException($"The TickType '{requestData.TickType}' is not supported in the {nameof(GetData)} method. Please implement the necessary logic for handling this TickType."); + var time = ConvertDateTimeToSpecificDateTimeZoneBySecurityType(requestData.Symbol.SecurityType, t.DateTimeStamp, requestData.DataTimeZone); + switch (requestData.TickType) + { + case TickType.Trade: + return new Tick() + { + Time = time, + Value = (decimal)t.Last, + DataType = MarketDataType.Tick, + Symbol = requestData.Symbol, + TickType = TickType.Trade, + Quantity = t.LastSize, + }; + case TickType.Quote: + return new Tick() + { + Time = time, + DataType = MarketDataType.Tick, + Symbol = requestData.Symbol, + TickType = TickType.Quote, + AskPrice = (decimal)t.Ask, + //AskSize = askSize, + BidPrice = (decimal)t.Bid, + //BidSize = bidSize, + }; + default: + throw new NotImplementedException($"The TickType '{requestData.TickType}' is not supported in the {nameof(GetData)} method. Please implement the necessary logic for handling this TickType."); + } } + return null; case LookupType.REQ_HST_INT: if (e is LookupIntervalEventArgs i && i.DateTimeStamp != default) { - var startTime = i.DateTimeStamp - requestData.Resolution; - if (!isEquity) - { - startTime = startTime.ConvertTo(TimeZoneIQFeed, requestData.SymbolDateTimeZone); - } - - return new TradeBar(startTime, requestData.Symbol, i.Open, i.High, i.Low, i.Close, i.PeriodVolume, requestData.Resolution); + var resolutionTimeSpan = requestData.Resolution.ToTimeSpan(); + var startTime = ConvertDateTimeToSpecificDateTimeZoneBySecurityType(requestData.Symbol.SecurityType, i.DateTimeStamp - resolutionTimeSpan, requestData.DataTimeZone); + return new TradeBar(startTime, requestData.Symbol, i.Open, i.High, i.Low, i.Close, i.PeriodVolume, resolutionTimeSpan); } return null; case LookupType.REQ_HST_DWM: - var d = (LookupDayWeekMonthEventArgs)e; - if (d.DateTimeStamp == DateTime.MinValue) return null; - var dstartTime = d.DateTimeStamp.Date; - if (!isEquity) dstartTime = dstartTime.ConvertTo(TimeZoneIQFeed, TimeZones.EasternStandard); - return new TradeBar(dstartTime, requestData.Symbol, (decimal)d.Open, (decimal)d.High, (decimal)d.Low, (decimal)d.Close, d.PeriodVolume, requestData.Resolution); - + if (e is LookupDayWeekMonthEventArgs d && d.DateTimeStamp != default) + { + var startTime = ConvertDateTimeToSpecificDateTimeZoneBySecurityType(requestData.Symbol.SecurityType, d.DateTimeStamp.Date, requestData.DataTimeZone); + return new TradeBar(startTime, requestData.Symbol, (decimal)d.Open, (decimal)d.High, (decimal)d.Low, (decimal)d.Close, d.PeriodVolume, requestData.Resolution.ToTimeSpan()); + } + return null; // we don't need to handle these other types case LookupType.REQ_SYM_SYM: case LookupType.REQ_SYM_SIC: @@ -1059,6 +1056,26 @@ private BaseData GetData(LookupEventArgs e, HistoryRequestContext requestData) } } + /// + /// Converts the specified to the given + /// based on the provided . If the security type is not + /// and the destination time zone is different from the IQFeed time zone, the conversion is performed. + /// + /// The type of the security (e.g., Equity, Future, Option). + /// The original value to convert. + /// The target for conversion. + /// + /// The converted if conditions are met; otherwise, returns the original . + /// + private static DateTime ConvertDateTimeToSpecificDateTimeZoneBySecurityType(SecurityType securityType, DateTime dateTime, DateTimeZone dateTimeZone) + { + if (securityType != SecurityType.Equity && TimeZoneIQFeed != dateTimeZone) + { + return dateTime.ConvertTo(TimeZoneIQFeed, dateTimeZone); + } + return dateTime; + } + private static PeriodType GetPeriodType(Resolution resolution) { switch (resolution) diff --git a/QuantConnect.IQFeed/Models/HistoryRequestContext.cs b/QuantConnect.IQFeed/Models/HistoryRequestContext.cs deleted file mode 100644 index 131cfb3..0000000 --- a/QuantConnect.IQFeed/Models/HistoryRequestContext.cs +++ /dev/null @@ -1,62 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using NodaTime; -using QuantConnect.Securities; - -namespace QuantConnect.Lean.DataSource.IQFeed.Models; - -/// -/// Represents a subset of parameters extracted from a history request, -/// providing context for historical data queries related to a specific symbol. -/// -public class HistoryRequestContext -{ - /// - /// Gets the symbol for which historical data is requested. - /// - public Symbol Symbol { get; } - - /// - /// Gets the type of tick data requested (e.g., trade, quote). - /// - public TickType TickType { get; } - - /// - /// Gets the resolution of the historical data request as a . - /// - public TimeSpan Resolution { get; } - - /// - /// Gets the time zone associated with the symbol's exchange. - /// - public DateTimeZone SymbolDateTimeZone { get; } - - /// - /// Initializes a new instance of the class, - /// setting symbol, tick type, resolution, and determining the symbol's exchange time zone. - /// - /// The symbol representing the financial instrument. - /// The tick data type (trade, quote, etc.). - /// The resolution of the requested historical data. - public HistoryRequestContext(Symbol symbol, TickType tickType, Resolution resolution) - { - Symbol = symbol; - TickType = tickType; - Resolution = resolution.ToTimeSpan(); - SymbolDateTimeZone = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - } -} From 237dd4d056cc6e94f0973ff0be5b84424092b214 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 1 Aug 2025 19:09:33 +0300 Subject: [PATCH 21/25] refactor: handle received history data --- .../IQFeedAPI/IQLookupHistorySymbolClient.cs | 92 ++++++++----------- QuantConnect.IQFeed/IQFeedDataProvider.cs | 71 +++++--------- 2 files changed, 59 insertions(+), 104 deletions(-) diff --git a/QuantConnect.IQFeed/IQFeedAPI/IQLookupHistorySymbolClient.cs b/QuantConnect.IQFeed/IQFeedAPI/IQLookupHistorySymbolClient.cs index 87165cc..eae9a08 100644 --- a/QuantConnect.IQFeed/IQFeedAPI/IQLookupHistorySymbolClient.cs +++ b/QuantConnect.IQFeed/IQFeedAPI/IQLookupHistorySymbolClient.cs @@ -23,8 +23,14 @@ namespace QuantConnect.Lean.DataSource.IQFeed // Historical stock data lookup events public class LookupTickEventArgs : LookupEventArgs { - public LookupTickEventArgs(string requestId, string line) : - base(requestId, LookupType.REQ_HST_TCK, LookupSequence.MessageDetail) + public DateTime DateTimeStamp { get; } + public decimal Last { get; } + public decimal LastSize { get; } + public decimal Bid { get; } + public decimal Ask { get; } + + public LookupTickEventArgs(string requestId, string line) + : base(requestId, LookupType.REQ_HST_TCK, LookupSequence.MessageDetail) { var fields = line.Split(','); if (fields.Length < 11) @@ -32,35 +38,14 @@ public LookupTickEventArgs(string requestId, string line) : Log.Error("LookupIntervalEventArgs.ctor(): " + line); return; } - if (!DateTime.TryParseExact(fields[1], "yyyy-MM-dd HH:mm:ss", _enUS, DateTimeStyles.None, out _dateTimeStamp)) _dateTimeStamp = DateTime.MinValue; - if (!double.TryParse(fields[2], out _last)) _last = 0; - if (!int.TryParse(fields[3], out _lastSize)) _lastSize = 0; - if (!int.TryParse(fields[4], out _totalVolume)) _totalVolume = 0; - if (!double.TryParse(fields[5], out _bid)) _bid = 0; - if (!double.TryParse(fields[6], out _ask)) _ask = 0; - if (!int.TryParse(fields[7], out _tickId)) _tickId = 0; - if (!char.TryParse(fields[10], out _basis)) _basis = ' '; - } - public DateTime DateTimeStamp { get { return _dateTimeStamp; } } - public double Last { get { return _last; } } - public int LastSize { get { return _lastSize; } } - public int TotalVolume { get { return _totalVolume; } } - public double Bid { get { return _bid; } } - public double Ask { get { return _ask; } } - public int TickId { get { return _tickId; } } - public char Basis { get { return _basis; } } - #region private - private DateTime _dateTimeStamp; - private double _last; - private int _lastSize; - private int _totalVolume; - private double _bid; - private double _ask; - private int _tickId; - private char _basis; - private CultureInfo _enUS = new CultureInfo("en-US"); - #endregion + DateTimeStamp = DateTime.TryParseExact(fields[1], "yyyy-MM-dd HH:mm:ss", new CultureInfo("en-US"), DateTimeStyles.None, out var dateTimeStamp) ? dateTimeStamp : default; + + Last = decimal.TryParse(fields[2], out var l) ? l : 0m; + LastSize = decimal.TryParse(fields[3], out var ls) ? ls : 0m; + Bid = decimal.TryParse(fields[5], out var b) ? b : 0m; + Ask = decimal.TryParse(fields[6], out var a) ? a : 0m; + } } public class LookupIntervalEventArgs : LookupEventArgs @@ -95,8 +80,16 @@ public LookupIntervalEventArgs(string requestId, string line) public class LookupDayWeekMonthEventArgs : LookupEventArgs { - public LookupDayWeekMonthEventArgs(string requestId, string line) : - base(requestId, LookupType.REQ_HST_DWM, LookupSequence.MessageDetail) + public DateTime DateTimeStamp { get; } + public decimal High { get; } + public decimal Low { get; } + public decimal Open { get; } + public decimal Close { get; } + public int PeriodVolume { get; } + public int OpenInterest { get; } + + public LookupDayWeekMonthEventArgs(string requestId, string line) + : base(requestId, LookupType.REQ_HST_DWM, LookupSequence.MessageDetail) { var fields = line.Split(','); if (fields.Length < 8) @@ -104,32 +97,19 @@ public LookupDayWeekMonthEventArgs(string requestId, string line) : Log.Error("LookupIntervalEventArgs.ctor(): " + line); return; } - if (!DateTime.TryParseExact(fields[1], "yyyy-MM-dd HH:mm:ss", _enUS, DateTimeStyles.None, out _dateTimeStamp)) _dateTimeStamp = DateTime.MinValue; - if (!double.TryParse(fields[2], out _high)) _high = 0; - if (!double.TryParse(fields[3], out _low)) _low = 0; - if (!double.TryParse(fields[4], out _open)) _open = 0; - if (!double.TryParse(fields[5], out _close)) _close = 0; - if (!int.TryParse(fields[6], out _periodVolume)) _periodVolume = 0; - if (!int.TryParse(fields[7], out _openInterest)) _openInterest = 0; + + + DateTimeStamp = DateTime.TryParseExact(fields[1], DateFormat.DB, new CultureInfo("en-US"), DateTimeStyles.None, out var dateTimeStamp) ? dateTimeStamp : default; + High = decimal.TryParse(fields[2], out var h) ? h : 0m; + Low = decimal.TryParse(fields[3], out var l) ? l : 0m; + Open = decimal.TryParse(fields[4], out var o) ? o : 0m; + Close = decimal.TryParse(fields[5], out var c) ? c : 0m; + PeriodVolume = int.TryParse(fields[6], out var p) ? p : 0; + OpenInterest = int.TryParse(fields[7], out var t) ? t : 0; } - public DateTime DateTimeStamp { get { return _dateTimeStamp; } } - public double High { get { return _high; } } - public double Low { get { return _low; } } - public double Open { get { return _open; } } - public double Close { get { return _close; } } - public int PeriodVolume { get { return _periodVolume; } } - public int OpenInterest { get { return _openInterest; } } - #region private - private DateTime _dateTimeStamp; - private double _high; - private double _low; - private double _open; - private double _close; - private int _periodVolume; - private int _openInterest; - private CultureInfo _enUS = new CultureInfo("en-US"); - #endregion + + } // Symbol search lookup events diff --git a/QuantConnect.IQFeed/IQFeedDataProvider.cs b/QuantConnect.IQFeed/IQFeedDataProvider.cs index 9733401..a6b5a85 100644 --- a/QuantConnect.IQFeed/IQFeedDataProvider.cs +++ b/QuantConnect.IQFeed/IQFeedDataProvider.cs @@ -764,7 +764,7 @@ public class HistoryPort : IQLookupHistorySymbolClient /// /// Represents the time zone used by IQFeed, which returns time in the New York (EST) Time Zone with daylight savings time. /// - private static DateTimeZone TimeZoneIQFeed = TimeZones.NewYork; + private readonly static DateTimeZone TimeZoneIQFeed = TimeZones.NewYork; /// /// Indicates whether the warning for invalid has been fired. @@ -845,7 +845,6 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo var ticker = _symbolUniverse.GetBrokerageSymbol(request.Symbol); var start = request.StartTimeUtc.ConvertFromUtc(TimeZoneIQFeed); DateTime? end = request.EndTimeUtc.ConvertFromUtc(TimeZoneIQFeed); - var exchangeTz = request.ExchangeHours.TimeZone; // if we're within a minute of now, don't set the end time if (request.EndTimeUtc >= DateTime.UtcNow.AddMinutes(-1)) { @@ -883,7 +882,7 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo continue; } - return GetSlice(exchangeTz); + return GetSlice(request.DataTimeZone); } private IEnumerable? GetSlice(DateTimeZone exchangeTz) @@ -895,6 +894,8 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo { foreach (var tradeBar in tradeBars) { + tradeBar.Time = tradeBar.Time.ConvertTo(TimeZoneIQFeed, exchangeTz); + tradeBar.EndTime = tradeBar.EndTime.ConvertTo(TimeZoneIQFeed, exchangeTz); // Returns IEnumerable object yield return new Slice(tradeBar.EndTime, new[] { tradeBar }, tradeBar.EndTime.ConvertToUtc(exchangeTz)); } @@ -990,49 +991,43 @@ private BaseData GetData(LookupEventArgs e, HistoryRequest requestData) case LookupType.REQ_HST_TCK: if (e is LookupTickEventArgs t) { - var time = ConvertDateTimeToSpecificDateTimeZoneBySecurityType(requestData.Symbol.SecurityType, t.DateTimeStamp, requestData.DataTimeZone); + var tick = new Tick() + { + Time = t.DateTimeStamp, + DataType = MarketDataType.Tick, + Symbol = requestData.Symbol, + }; + switch (requestData.TickType) { case TickType.Trade: - return new Tick() - { - Time = time, - Value = (decimal)t.Last, - DataType = MarketDataType.Tick, - Symbol = requestData.Symbol, - TickType = TickType.Trade, - Quantity = t.LastSize, - }; + tick.TickType = TickType.Trade; + tick.Value = t.Last; + tick.Quantity = t.LastSize; + break; case TickType.Quote: - return new Tick() - { - Time = time, - DataType = MarketDataType.Tick, - Symbol = requestData.Symbol, - TickType = TickType.Quote, - AskPrice = (decimal)t.Ask, - //AskSize = askSize, - BidPrice = (decimal)t.Bid, - //BidSize = bidSize, - }; + tick.TickType = TickType.Quote; + tick.AskPrice = t.Ask; + tick.BidPrice = t.Bid; + break; default: throw new NotImplementedException($"The TickType '{requestData.TickType}' is not supported in the {nameof(GetData)} method. Please implement the necessary logic for handling this TickType."); } + + return tick; } return null; case LookupType.REQ_HST_INT: if (e is LookupIntervalEventArgs i && i.DateTimeStamp != default) { var resolutionTimeSpan = requestData.Resolution.ToTimeSpan(); - var startTime = ConvertDateTimeToSpecificDateTimeZoneBySecurityType(requestData.Symbol.SecurityType, i.DateTimeStamp - resolutionTimeSpan, requestData.DataTimeZone); - return new TradeBar(startTime, requestData.Symbol, i.Open, i.High, i.Low, i.Close, i.PeriodVolume, resolutionTimeSpan); + return new TradeBar(i.DateTimeStamp - resolutionTimeSpan, requestData.Symbol, i.Open, i.High, i.Low, i.Close, i.PeriodVolume, resolutionTimeSpan); } return null; case LookupType.REQ_HST_DWM: if (e is LookupDayWeekMonthEventArgs d && d.DateTimeStamp != default) { - var startTime = ConvertDateTimeToSpecificDateTimeZoneBySecurityType(requestData.Symbol.SecurityType, d.DateTimeStamp.Date, requestData.DataTimeZone); - return new TradeBar(startTime, requestData.Symbol, (decimal)d.Open, (decimal)d.High, (decimal)d.Low, (decimal)d.Close, d.PeriodVolume, requestData.Resolution.ToTimeSpan()); + return new TradeBar(d.DateTimeStamp.Date, requestData.Symbol, d.Open, d.High, d.Low, d.Close, d.PeriodVolume, requestData.Resolution.ToTimeSpan()); } return null; // we don't need to handle these other types @@ -1056,26 +1051,6 @@ private BaseData GetData(LookupEventArgs e, HistoryRequest requestData) } } - /// - /// Converts the specified to the given - /// based on the provided . If the security type is not - /// and the destination time zone is different from the IQFeed time zone, the conversion is performed. - /// - /// The type of the security (e.g., Equity, Future, Option). - /// The original value to convert. - /// The target for conversion. - /// - /// The converted if conditions are met; otherwise, returns the original . - /// - private static DateTime ConvertDateTimeToSpecificDateTimeZoneBySecurityType(SecurityType securityType, DateTime dateTime, DateTimeZone dateTimeZone) - { - if (securityType != SecurityType.Equity && TimeZoneIQFeed != dateTimeZone) - { - return dateTime.ConvertTo(TimeZoneIQFeed, dateTimeZone); - } - return dateTime; - } - private static PeriodType GetPeriodType(Resolution resolution) { switch (resolution) From 6d3c4bd9609ba2dcba5ea6be05843e864ab6f809 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 1 Aug 2025 20:11:32 +0300 Subject: [PATCH 22/25] refactor: use ConvertTo(IQFeedDataProvider.TimeZoneIQFeed, exchangeTimeZone) --- QuantConnect.IQFeed/IQFeedDataProvider.cs | 130 ++++++++-------------- 1 file changed, 49 insertions(+), 81 deletions(-) diff --git a/QuantConnect.IQFeed/IQFeedDataProvider.cs b/QuantConnect.IQFeed/IQFeedDataProvider.cs index a6b5a85..e17f9bb 100644 --- a/QuantConnect.IQFeed/IQFeedDataProvider.cs +++ b/QuantConnect.IQFeed/IQFeedDataProvider.cs @@ -22,14 +22,13 @@ using QuantConnect.Api; using QuantConnect.Util; using QuantConnect.Data; -using System.Diagnostics; using Newtonsoft.Json.Linq; using QuantConnect.Packets; using QuantConnect.Logging; +using QuantConnect.Securities; using QuantConnect.Interfaces; using QuantConnect.Data.Market; using QuantConnect.Configuration; -using Timer = System.Timers.Timer; using System.Security.Cryptography; using System.Collections.Concurrent; using System.Net.NetworkInformation; @@ -48,6 +47,11 @@ public class IQFeedDataProvider : HistoryProviderBase, IDataQueueHandler, IDataQ private readonly object _sync = new object(); private IQFeedDataQueueUniverseProvider _symbolUniverse; + /// + /// Represents the time zone used by IQFeed, which returns time in the New York (EST) Time Zone with daylight savings time. + /// + public readonly static DateTimeZone TimeZoneIQFeed = TimeZones.NewYork; + //Socket connections: private AdminPort _adminPort; private Level1Port _level1Port; @@ -233,7 +237,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) var subscriptions = new List>(); foreach (var request in requests) { - var history = _historyPort.ProcessHistoryRequests(request); + var history = _historyPort.ProcessHistoryRequests(request, sliceTimeZone); if (history == null) { @@ -556,66 +560,53 @@ public AdminPort() /// public class Level1Port : IQLevel1Client { - private int count; - private DateTime start; - private DateTime _feedTime; - private Stopwatch _stopwatch = new Stopwatch(); - private readonly Timer _timer; private readonly ConcurrentDictionary _prices; private readonly ConcurrentDictionary _openInterests; private readonly IQFeedDataQueueUniverseProvider _symbolUniverse; private readonly IDataAggregator _aggregator; - private int _dataQueueCount; - public DateTime FeedTime - { - get - { - if (_feedTime == new DateTime()) return DateTime.Now; - return _feedTime.AddMilliseconds(_stopwatch.ElapsedMilliseconds); - } - set - { - _feedTime = value; - _stopwatch = Stopwatch.StartNew(); - } - } + /// + /// A thread-safe dictionary that maps a to a . + /// + /// + /// This dictionary is used to store the time zone information for each symbol in a concurrent environment, + /// ensuring thread safety when accessing or modifying the time zone data. + /// + private readonly ConcurrentDictionary _exchangeTimeZoneByLeanSymbol = new(); public Level1Port(IDataAggregator aggregator, IQFeedDataQueueUniverseProvider symbolUniverse) : base(IQFeedDefault.BufferSize) { - start = DateTime.Now; _prices = new ConcurrentDictionary(); _openInterests = new ConcurrentDictionary(); _aggregator = aggregator; _symbolUniverse = symbolUniverse; Level1SummaryUpdateEvent += OnLevel1SummaryUpdateEvent; - Level1TimerEvent += OnLevel1TimerEvent; Level1ServerDisconnectedEvent += OnLevel1ServerDisconnected; Level1ServerReconnectFailed += OnLevel1ServerReconnectFailed; Level1UnknownEvent += OnLevel1UnknownEvent; Level1FundamentalEvent += OnLevel1FundamentalEvent; + } - _timer = new Timer(1000); - _timer.Enabled = false; - _timer.AutoReset = true; - _timer.Elapsed += (sender, args) => + /// + /// Returns a timestamp for a tick converted to the exchange time zone + /// + private DateTime GetRealTimeTickTime(Symbol symbol) + { + var time = DateTime.UtcNow; + var exchangeTimeZone = default(DateTimeZone); + lock (_exchangeTimeZoneByLeanSymbol) { - var ticksPerSecond = count / (DateTime.Now - start).TotalSeconds; - int dataQueueCount = Interlocked.Exchange(ref _dataQueueCount, 0); - if (ticksPerSecond > 1000 || dataQueueCount > 31) + if (!_exchangeTimeZoneByLeanSymbol.TryGetValue(symbol, out exchangeTimeZone)) { - Log.Trace($"IQFeed.OnSecond(): Ticks/sec: {ticksPerSecond.ToStringInvariant("0000.00")} " + - $"Engine.Ticks.Count: {dataQueueCount} CPU%: {OS.CpuUsage.ToStringInvariant("0.0") + "%"}" - ); + // read the exchange time zone from market-hours-database + exchangeTimeZone = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; + _exchangeTimeZoneByLeanSymbol[symbol] = exchangeTimeZone; } + } - count = 0; - start = DateTime.Now; - }; - - _timer.Enabled = true; + return time.ConvertTo(IQFeedDataProvider.TimeZoneIQFeed, exchangeTimeZone); } private Symbol GetLeanSymbol(string ticker) @@ -635,7 +626,7 @@ private void OnLevel1FundamentalEvent(object sender, Level1FundamentalEventArgs _prices.TryGetValue(e.Symbol, out referencePrice); var symbol = GetLeanSymbol(e.Symbol); - var split = new Data.Market.Split(symbol, FeedTime, (decimal)referencePrice, (decimal)e.SplitFactor1, SplitType.SplitOccurred); + var split = new Data.Market.Split(symbol, GetRealTimeTickTime(symbol), (decimal)referencePrice, (decimal)e.SplitFactor1, SplitType.SplitOccurred); Emit(split); } } @@ -657,29 +648,20 @@ private void OnLevel1SummaryUpdateEvent(object sender, Level1SummaryUpdateEventA && e.TypeOfUpdate != Level1SummaryUpdateEventArgs.UpdateType.Bid && e.TypeOfUpdate != Level1SummaryUpdateEventArgs.UpdateType.Ask) return; - count++; - var time = FeedTime; var last = (decimal)(e.TypeOfUpdate == Level1SummaryUpdateEventArgs.UpdateType.ExtendedTrade ? e.ExtendedTradingLast : e.Last); var symbol = GetLeanSymbol(e.Symbol); + var time = GetRealTimeTickTime(symbol); TickType tradeType; switch (symbol.ID.SecurityType) { - // the feed time is in NYC/EDT, convert it into EST case SecurityType.Forex: - - time = FeedTime.ConvertTo(TimeZones.NewYork, TimeZones.EasternStandard); // TypeOfUpdate always equal to UpdateType.Trade for FXCM, but the message contains B/A and last data tradeType = TickType.Quote; - break; - - // for all other asset classes we leave it as is (NYC/EDT) default: - - time = FeedTime; tradeType = e.TypeOfUpdate == Level1SummaryUpdateEventArgs.UpdateType.Bid || e.TypeOfUpdate == Level1SummaryUpdateEventArgs.UpdateType.Ask ? TickType.Quote : @@ -687,7 +669,7 @@ private void OnLevel1SummaryUpdateEvent(object sender, Level1SummaryUpdateEventA break; } - var tick = new Tick(time, symbol, last, (decimal)e.Bid, (decimal)e.Ask) + var tick = new Tick(GetRealTimeTickTime(symbol), symbol, last, (decimal)e.Bid, (decimal)e.Ask) { AskSize = e.AskSize, BidSize = e.BidSize, @@ -713,19 +695,6 @@ private void OnLevel1SummaryUpdateEvent(object sender, Level1SummaryUpdateEventA private void Emit(BaseData tick) { _aggregator.Update(tick); - Interlocked.Increment(ref _dataQueueCount); - } - - /// - /// Set the interal clock time. - /// - private void OnLevel1TimerEvent(object sender, Level1TimerEventArgs e) - { - //If there was a bad tick and the time didn't set right, skip setting it here and just use our millisecond timer to set the time from last time it was set. - if (e.DateTimeStamp != DateTime.MinValue) - { - FeedTime = e.DateTimeStamp; - } } /// @@ -761,11 +730,6 @@ public class HistoryPort : IQLookupHistorySymbolClient private ConcurrentDictionary> _currentRequest; private readonly IQFeedDataQueueUniverseProvider _symbolUniverse; - /// - /// Represents the time zone used by IQFeed, which returns time in the New York (EST) Time Zone with daylight savings time. - /// - private readonly static DateTimeZone TimeZoneIQFeed = TimeZones.NewYork; - /// /// Indicates whether the warning for invalid has been fired. /// @@ -802,9 +766,14 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo } /// - /// Populate request data + /// Processes the specified historical data request and generates a sequence of instances. /// - public IEnumerable? ProcessHistoryRequests(HistoryRequest request) + /// The historical data requests + /// The time zone used when time stamping the slice instances + /// An enumerable sequence of objects representing the historical data for the request, + /// or null if no data could be retrieved or processed. + /// + public IEnumerable? ProcessHistoryRequests(HistoryRequest request, DateTimeZone sliceTimeZone) { // skipping universe and canonical symbols if (!CanHandle(request.Symbol) || @@ -843,8 +812,8 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo _inProgress = true; var ticker = _symbolUniverse.GetBrokerageSymbol(request.Symbol); - var start = request.StartTimeUtc.ConvertFromUtc(TimeZoneIQFeed); - DateTime? end = request.EndTimeUtc.ConvertFromUtc(TimeZoneIQFeed); + var start = request.StartTimeUtc.ConvertFromUtc(IQFeedDataProvider.TimeZoneIQFeed); + DateTime? end = request.EndTimeUtc.ConvertFromUtc(IQFeedDataProvider.TimeZoneIQFeed); // if we're within a minute of now, don't set the end time if (request.EndTimeUtc >= DateTime.UtcNow.AddMinutes(-1)) { @@ -882,10 +851,10 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo continue; } - return GetSlice(request.DataTimeZone); + return GetSlice(request.ExchangeHours.TimeZone, sliceTimeZone); } - private IEnumerable? GetSlice(DateTimeZone exchangeTz) + private IEnumerable? GetSlice(DateTimeZone exchangeTz, DateTimeZone sliceTimeZone) { // After all data arrive, we pass it to the algorithm through memory and write to a file foreach (var key in _currentRequest.Keys) @@ -894,10 +863,8 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo { foreach (var tradeBar in tradeBars) { - tradeBar.Time = tradeBar.Time.ConvertTo(TimeZoneIQFeed, exchangeTz); - tradeBar.EndTime = tradeBar.EndTime.ConvertTo(TimeZoneIQFeed, exchangeTz); // Returns IEnumerable object - yield return new Slice(tradeBar.EndTime, new[] { tradeBar }, tradeBar.EndTime.ConvertToUtc(exchangeTz)); + yield return new Slice(tradeBar.EndTime.ConvertTo(exchangeTz, sliceTimeZone), new[] { tradeBar }, tradeBar.EndTime.ConvertToUtc(exchangeTz)); } } } @@ -993,7 +960,7 @@ private BaseData GetData(LookupEventArgs e, HistoryRequest requestData) { var tick = new Tick() { - Time = t.DateTimeStamp, + Time = t.DateTimeStamp.ConvertTo(IQFeedDataProvider.TimeZoneIQFeed, requestData.ExchangeHours.TimeZone), DataType = MarketDataType.Tick, Symbol = requestData.Symbol, }; @@ -1021,13 +988,14 @@ private BaseData GetData(LookupEventArgs e, HistoryRequest requestData) if (e is LookupIntervalEventArgs i && i.DateTimeStamp != default) { var resolutionTimeSpan = requestData.Resolution.ToTimeSpan(); - return new TradeBar(i.DateTimeStamp - resolutionTimeSpan, requestData.Symbol, i.Open, i.High, i.Low, i.Close, i.PeriodVolume, resolutionTimeSpan); + var time = (i.DateTimeStamp - resolutionTimeSpan).ConvertTo(IQFeedDataProvider.TimeZoneIQFeed, requestData.ExchangeHours.TimeZone); + return new TradeBar(time, requestData.Symbol, i.Open, i.High, i.Low, i.Close, i.PeriodVolume, resolutionTimeSpan); } return null; case LookupType.REQ_HST_DWM: if (e is LookupDayWeekMonthEventArgs d && d.DateTimeStamp != default) { - return new TradeBar(d.DateTimeStamp.Date, requestData.Symbol, d.Open, d.High, d.Low, d.Close, d.PeriodVolume, requestData.Resolution.ToTimeSpan()); + return new TradeBar(d.DateTimeStamp.Date.ConvertTo(IQFeedDataProvider.TimeZoneIQFeed, requestData.ExchangeHours.TimeZone), requestData.Symbol, d.Open, d.High, d.Low, d.Close, d.PeriodVolume, requestData.Resolution.ToTimeSpan()); } return null; // we don't need to handle these other types From 70d42ff718f46a174b6c92a3727d883663afbf49 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 1 Aug 2025 20:18:44 +0300 Subject: [PATCH 23/25] test:feat: Data Queue Handler --- .../IQFeedDataProviderTests.cs | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/QuantConnect.IQFeed.Tests/IQFeedDataProviderTests.cs b/QuantConnect.IQFeed.Tests/IQFeedDataProviderTests.cs index 03e7728..67babf6 100644 --- a/QuantConnect.IQFeed.Tests/IQFeedDataProviderTests.cs +++ b/QuantConnect.IQFeed.Tests/IQFeedDataProviderTests.cs @@ -22,6 +22,7 @@ using QuantConnect.Tests; using QuantConnect.Logging; using System.Threading.Tasks; +using QuantConnect.Securities; using QuantConnect.Data.Market; using System.Collections.Generic; using QuantConnect.Lean.Engine.DataFeeds.Enumerators; @@ -52,6 +53,11 @@ private static IEnumerable SubscribeTestCaseData yield return new TestCaseData(Symbols.AAPL); yield return new TestCaseData(Symbol.Create("SMCI", SecurityType.Equity, Market.USA)); yield return new TestCaseData(Symbol.Create("IRBT", SecurityType.Equity, Market.USA)); + var nasdaq100EMini = Symbol.CreateFuture(Futures.Indices.NASDAQ100EMini, Market.CME, new DateTime(2025, 09, 19)); + yield return new TestCaseData(nasdaq100EMini); + var naturalGasAug2025 = Symbol.CreateFuture(Futures.Energy.NaturalGas, Market.NYMEX, new DateTime(2025, 08, 27)); + yield return new TestCaseData(naturalGasAug2025); + // yield return new TestCaseData(Symbols.SPY); // Not supported. @@ -127,19 +133,22 @@ private void SubscribeOnData(Symbol symbol, Resolution resolution, int minimumRe Action callback = (dataPoint) => { - if (dataPoint == null) + if (dataPoint is null) { return; } - switch (dataPoint) + if (dataPoint is Tick tick) { - case TradeBar _: - secondDataReceived[typeof(TradeBar)] += 1; - break; - case QuoteBar _: - secondDataReceived[typeof(QuoteBar)] += 1; - break; + switch (tick.TickType) + { + case TickType.Trade: + secondDataReceived[typeof(TradeBar)] += 1; + break; + case TickType.Quote: + secondDataReceived[typeof(QuoteBar)] += 1; + break; + } } if (secondDataReceived.All(type => type.Value >= minimumReturnDataAmount)) From 883aa87b52efd035ef396686148f458d8e00b349 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 1 Aug 2025 21:49:37 +0300 Subject: [PATCH 24/25] fix: use convertFromUtc instead of ConvertTo in DQH --- QuantConnect.IQFeed/IQFeedDataProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/QuantConnect.IQFeed/IQFeedDataProvider.cs b/QuantConnect.IQFeed/IQFeedDataProvider.cs index e17f9bb..7c053c2 100644 --- a/QuantConnect.IQFeed/IQFeedDataProvider.cs +++ b/QuantConnect.IQFeed/IQFeedDataProvider.cs @@ -606,7 +606,7 @@ private DateTime GetRealTimeTickTime(Symbol symbol) } } - return time.ConvertTo(IQFeedDataProvider.TimeZoneIQFeed, exchangeTimeZone); + return time.ConvertFromUtc(exchangeTimeZone); } private Symbol GetLeanSymbol(string ticker) From 6393747eee3909ff68443e6fe89e96f16615db2c Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 1 Aug 2025 23:35:58 +0300 Subject: [PATCH 25/25] fix: history request --- QuantConnect.IQFeed/IQFeedDataProvider.cs | 29 ++++++++++++----------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/QuantConnect.IQFeed/IQFeedDataProvider.cs b/QuantConnect.IQFeed/IQFeedDataProvider.cs index 7c053c2..a023823 100644 --- a/QuantConnect.IQFeed/IQFeedDataProvider.cs +++ b/QuantConnect.IQFeed/IQFeedDataProvider.cs @@ -32,6 +32,8 @@ using System.Security.Cryptography; using System.Collections.Concurrent; using System.Net.NetworkInformation; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Lean.Engine.HistoricalData; using HistoryRequest = QuantConnect.Data.HistoryRequest; namespace QuantConnect.Lean.DataSource.IQFeed @@ -39,7 +41,7 @@ namespace QuantConnect.Lean.DataSource.IQFeed /// /// IQFeedDataProvider is an implementation of IDataQueueHandler and IHistoryProvider /// - public class IQFeedDataProvider : HistoryProviderBase, IDataQueueHandler, IDataQueueUniverseProvider + public class IQFeedDataProvider : SynchronizingHistoryProvider, IDataQueueHandler, IDataQueueUniverseProvider { private bool _isConnected; private readonly HashSet _symbols; @@ -234,7 +236,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) /// An enumerable of the slices of data covering the span specified in each request public override IEnumerable? GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) { - var subscriptions = new List>(); + var subscriptions = new List(); foreach (var request in requests) { var history = _historyPort.ProcessHistoryRequests(request, sliceTimeZone); @@ -244,7 +246,8 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) continue; } - subscriptions.Add(history); + var subscription = CreateSubscription(request, history); + subscriptions.Add(subscription); } if (subscriptions.Count == 0) @@ -252,7 +255,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return null; } - return subscriptions.SelectMany(x => x); + return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); } /// @@ -725,7 +728,7 @@ private void OnLevel1UnknownEvent(object sender, Level1TextLineEventArgs e) // this type is expected to be used for exactly one job at a time public class HistoryPort : IQLookupHistorySymbolClient { - private bool _inProgress; + private AutoResetEvent _historyRequestResetEvent = new(false); private readonly ConcurrentDictionary _requestDataByRequestId = []; private ConcurrentDictionary> _currentRequest; private readonly IQFeedDataQueueUniverseProvider _symbolUniverse; @@ -773,7 +776,7 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo /// An enumerable sequence of objects representing the historical data for the request, /// or null if no data could be retrieved or processed. /// - public IEnumerable? ProcessHistoryRequests(HistoryRequest request, DateTimeZone sliceTimeZone) + public IEnumerable? ProcessHistoryRequests(HistoryRequest request, DateTimeZone sliceTimeZone) { // skipping universe and canonical symbols if (!CanHandle(request.Symbol) || @@ -808,9 +811,6 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo return null; } - // Set this process status - _inProgress = true; - var ticker = _symbolUniverse.GetBrokerageSymbol(request.Symbol); var start = request.StartTimeUtc.ConvertFromUtc(IQFeedDataProvider.TimeZoneIQFeed); DateTime? end = request.EndTimeUtc.ConvertFromUtc(IQFeedDataProvider.TimeZoneIQFeed); @@ -846,15 +846,16 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo _requestDataByRequestId[reqid] = request; - while (_inProgress) + if (!_historyRequestResetEvent.WaitOne(TimeSpan.FromSeconds(20))) { - continue; + Log.Trace($"{nameof(IQFeedDataProvider)}.{nameof(ProcessHistoryRequests)}: Timeout waiting for history data. Request details - Symbol: {request.Symbol.Value} ({request.Symbol.SecurityType}), Resolution: {request.Resolution}, StartTimeUtc: {request.StartTimeUtc:u}, EndTimeUtc: {request.EndTimeUtc:u}"); + return null; } return GetSlice(request.ExchangeHours.TimeZone, sliceTimeZone); } - private IEnumerable? GetSlice(DateTimeZone exchangeTz, DateTimeZone sliceTimeZone) + private IEnumerable? GetSlice(DateTimeZone exchangeTz, DateTimeZone sliceTimeZone) { // After all data arrive, we pass it to the algorithm through memory and write to a file foreach (var key in _currentRequest.Keys) @@ -864,7 +865,7 @@ public HistoryPort(IQFeedDataQueueUniverseProvider symbolUniverse, int maxDataPo foreach (var tradeBar in tradeBars) { // Returns IEnumerable object - yield return new Slice(tradeBar.EndTime.ConvertTo(exchangeTz, sliceTimeZone), new[] { tradeBar }, tradeBar.EndTime.ConvertToUtc(exchangeTz)); + yield return tradeBar; } } } @@ -917,7 +918,7 @@ protected override void OnLookupEvent(LookupEventArgs e) } break; case LookupSequence.MessageEnd: - _inProgress = false; + _historyRequestResetEvent.Set(); break; default: throw new ArgumentOutOfRangeException();